{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "convnet-vgg16.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "toc": {
      "nav_menu": {},
      "number_sections": true,
      "sideBar": true,
      "skip_h1_title": false,
      "title_cell": "Table of Contents",
      "title_sidebar": "Contents",
      "toc_cell": true,
      "toc_position": {
        "height": "calc(100% - 180px)",
        "left": "10px",
        "top": "150px",
        "width": "371px"
      },
      "toc_section_display": true,
      "toc_window_display": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "UEBilEjLj5wY"
      },
      "source": [
        "Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
        "- Author: Sebastian Raschka\n",
        "- GitHub Repository: https://github.com/rasbt/deeplearning-models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9G2nIyo6eA7A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install -q IPython\n",
        "!pip install -q ipykernel\n",
        "!pip install -q watermark\n",
        "!pip install -q matplotlib\n",
        "!pip install -q sklearn\n",
        "!pip install -q pandas\n",
        "!pip install -q pydot\n",
        "!pip install -q hiddenlayer\n",
        "!pip install -q graphviz"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "GOzuY8Yvj5wb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 155
        },
        "outputId": "24d4b0ac-b9bd-4c9a-a77c-92635481610d"
      },
      "source": [
        "%load_ext watermark\n",
        "%watermark -a 'Sebastian Raschka' -v -p torch"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The watermark extension is already loaded. To reload it, use:\n",
            "  %reload_ext watermark\n",
            "Sebastian Raschka \n",
            "\n",
            "CPython 3.6.9\n",
            "IPython 5.5.0\n",
            "\n",
            "torch 1.5.1+cu101\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "rH4XmErYj5wm"
      },
      "source": [
        "# AlexNet CIFAR-10 Classifier"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ze3pvj0Vd4eA",
        "colab_type": "text"
      },
      "source": [
        "### Network Architecture"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fEvyRRz_d4eB",
        "colab_type": "text"
      },
      "source": [
        "References\n",
        "    \n",
        "- [1] Krizhevsky, Alex, Ilya Sutskever, and Geoffrey E. Hinton. \"[Imagenet classification with deep convolutional neural networks.](https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf)\" In Advances in Neural Information Processing Systems, pp. 1097-1105. 2012.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "MkoGLH_Tj5wn"
      },
      "source": [
        "## Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ORj09gnrj5wp",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import time\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.utils.data.dataset import Subset\n",
        "\n",
        "from torchvision import datasets\n",
        "from torchvision import transforms\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "from PIL import Image\n",
        "\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "    torch.backends.cudnn.deterministic = True"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "I6hghKPxj5w0"
      },
      "source": [
        "## Model Settings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "NnT0sZIwj5wu",
        "colab": {}
      },
      "source": [
        "##########################\n",
        "### SETTINGS\n",
        "##########################\n",
        "\n",
        "# Hyperparameters\n",
        "RANDOM_SEED = 1\n",
        "LEARNING_RATE = 0.0001\n",
        "BATCH_SIZE = 256\n",
        "NUM_EPOCHS = 40\n",
        "\n",
        "# Architecture\n",
        "NUM_CLASSES = 10\n",
        "\n",
        "# Other\n",
        "DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "swagPqSUd4eJ",
        "colab_type": "text"
      },
      "source": [
        "## Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jmHeAjRTd4eK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "30682f34-ff85-4275-e16f-7d0fba809704"
      },
      "source": [
        "train_indices = torch.arange(0, 48000)\n",
        "valid_indices = torch.arange(48000, 50000)\n",
        "\n",
        "\n",
        "train_transform = transforms.Compose([transforms.Resize((70, 70)),\n",
        "                                      transforms.RandomCrop((64, 64)),\n",
        "                                      transforms.ToTensor()])\n",
        "\n",
        "test_transform = transforms.Compose([transforms.Resize((70, 70)),\n",
        "                                     transforms.CenterCrop((64, 64)),\n",
        "                                     transforms.ToTensor()])\n",
        "\n",
        "train_and_valid = datasets.CIFAR10(root='data', \n",
        "                                   train=True, \n",
        "                                   transform=train_transform,\n",
        "                                   download=True)\n",
        "\n",
        "train_dataset = Subset(train_and_valid, train_indices)\n",
        "valid_dataset = Subset(train_and_valid, valid_indices)\n",
        "test_dataset = datasets.CIFAR10(root='data', \n",
        "                                train=False, \n",
        "                                transform=test_transform,\n",
        "                                download=False)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "train_loader = DataLoader(dataset=train_dataset, \n",
        "                          batch_size=BATCH_SIZE,\n",
        "                          num_workers=4,\n",
        "                          shuffle=True)\n",
        "\n",
        "valid_loader = DataLoader(dataset=valid_dataset, \n",
        "                          batch_size=BATCH_SIZE,\n",
        "                          num_workers=4,\n",
        "                          shuffle=False)\n",
        "\n",
        "test_loader = DataLoader(dataset=test_dataset, \n",
        "                         batch_size=BATCH_SIZE,\n",
        "                         num_workers=4,\n",
        "                         shuffle=False)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Files already downloaded and verified\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qn1onaJud4eN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 225
        },
        "outputId": "3388049b-2f0c-4e0a-cebb-a01e705168f0"
      },
      "source": [
        "# Checking the dataset\n",
        "print('Training Set:\\n')\n",
        "for images, labels in train_loader:  \n",
        "    print('Image batch dimensions:', images.size())\n",
        "    print('Image label dimensions:', labels.size())\n",
        "    break\n",
        "    \n",
        "# Checking the dataset\n",
        "print('\\nValidation Set:')\n",
        "for images, labels in valid_loader:  \n",
        "    print('Image batch dimensions:', images.size())\n",
        "    print('Image label dimensions:', labels.size())\n",
        "    break\n",
        "\n",
        "# Checking the dataset\n",
        "print('\\nTesting Set:')\n",
        "for images, labels in train_loader:  \n",
        "    print('Image batch dimensions:', images.size())\n",
        "    print('Image label dimensions:', labels.size())\n",
        "    break"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training Set:\n",
            "\n",
            "Image batch dimensions: torch.Size([256, 3, 64, 64])\n",
            "Image label dimensions: torch.Size([256])\n",
            "\n",
            "Validation Set:\n",
            "Image batch dimensions: torch.Size([256, 3, 64, 64])\n",
            "Image label dimensions: torch.Size([256])\n",
            "\n",
            "Testing Set:\n",
            "Image batch dimensions: torch.Size([256, 3, 64, 64])\n",
            "Image label dimensions: torch.Size([256])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h0BL0pO8d4eP",
        "colab_type": "text"
      },
      "source": [
        "## Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bmfuK3wQd4eP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "##########################\n",
        "### MODEL\n",
        "##########################\n",
        "\n",
        "class AlexNet(nn.Module):\n",
        "\n",
        "    def __init__(self, num_classes):\n",
        "        super(AlexNet, self).__init__()\n",
        "        self.features = nn.Sequential(\n",
        "            nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=2),\n",
        "            nn.ReLU(inplace=True),\n",
        "            nn.MaxPool2d(kernel_size=3, stride=2),\n",
        "            nn.Conv2d(64, 192, kernel_size=5, padding=2),\n",
        "            nn.ReLU(inplace=True),\n",
        "            nn.MaxPool2d(kernel_size=3, stride=2),\n",
        "            nn.Conv2d(192, 384, kernel_size=3, padding=1),\n",
        "            nn.ReLU(inplace=True),\n",
        "            nn.Conv2d(384, 256, kernel_size=3, padding=1),\n",
        "            nn.ReLU(inplace=True),\n",
        "            nn.Conv2d(256, 256, kernel_size=3, padding=1),\n",
        "            nn.ReLU(inplace=True),\n",
        "            nn.MaxPool2d(kernel_size=3, stride=2),\n",
        "        )\n",
        "        self.avgpool = nn.AdaptiveAvgPool2d((6, 6))\n",
        "        self.classifier = nn.Sequential(\n",
        "            nn.Dropout(0.5),\n",
        "            nn.Linear(256 * 6 * 6, 4096),\n",
        "            nn.ReLU(inplace=True),\n",
        "            nn.Dropout(0.5),\n",
        "            nn.Linear(4096, 4096),\n",
        "            nn.ReLU(inplace=True),\n",
        "            nn.Linear(4096, num_classes)\n",
        "        )\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.features(x)\n",
        "        x = self.avgpool(x)\n",
        "        x = x.view(x.size(0), 256 * 6 * 6)\n",
        "        logits = self.classifier(x)\n",
        "        probas = F.softmax(logits, dim=1)\n",
        "        return logits, probas\n"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "_lza9t_uj5w1",
        "colab": {}
      },
      "source": [
        "torch.manual_seed(RANDOM_SEED)\n",
        "\n",
        "model = AlexNet(NUM_CLASSES)\n",
        "model.to(DEVICE)\n",
        "\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)  "
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bZaYBpGhePE-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 408
        },
        "outputId": "7acde0fe-a620-405c-8012-90f721e675e6"
      },
      "source": [
        "import hiddenlayer as hl\n",
        "hl.build_graph(model, torch.zeros([256, 3, 64, 64]).to(DEVICE))"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/onnx/symbolic_helper.py:176: UserWarning: ONNX export failed on adaptive_avg_pool2d because output size that are not factor of input size not supported\n",
            "  warnings.warn(\"ONNX export failed on \" + op + \" because \" + msg + \" not supported\")\n",
            "/usr/local/lib/python3.6/dist-packages/torch/onnx/symbolic_opset9.py:1310: UserWarning: Dropout is a training op and should not be exported in inference mode. Make sure to call eval() on the model, and to export it with param training=False.\n",
            "  warnings.warn(\"Dropout is a training op and should not be exported in inference mode. \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<hiddenlayer.graph.Graph at 0x7f869ff73390>"
            ],
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n -->\n<!-- Title: %3 Pages: 1 -->\n<svg width=\"2711pt\" height=\"223pt\"\n viewBox=\"0.00 0.00 2711.00 223.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(72 187)\">\n<title>%3</title>\n<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-72,36 -72,-187 2639,-187 2639,36 -72,36\"/>\n<!-- /outputs/19 -->\n<g id=\"node1\" class=\"node\">\n<title>/outputs/19</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"265,-40 195,-40 195,-4 265,-4 265,-40\"/>\n<text text-anchor=\"start\" x=\"203\" y=\"-19\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">MaxPool3x3</text>\n</g>\n<!-- 4813633701811250313 -->\n<g id=\"node19\" class=\"node\">\n<title>4813633701811250313</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"438,-40 354,-40 354,-4 438,-4 438,-40\"/>\n<text text-anchor=\"start\" x=\"362\" y=\"-19\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv5x5 &gt; Relu</text>\n</g>\n<!-- /outputs/19&#45;&gt;4813633701811250313 -->\n<g id=\"edge15\" class=\"edge\">\n<title>/outputs/19&#45;&gt;4813633701811250313</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M265.2043,-22C287.9563,-22 317.9768,-22 343.518,-22\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"343.7892,-25.5001 353.7892,-22 343.7891,-18.5001 343.7892,-25.5001\"/>\n<text text-anchor=\"middle\" x=\"309.5\" y=\"-25\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">256x64x7x7</text>\n</g>\n<!-- /outputs/22 -->\n<g id=\"node2\" class=\"node\">\n<title>/outputs/22</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"602,-40 532,-40 532,-4 602,-4 602,-40\"/>\n<text text-anchor=\"start\" x=\"540\" y=\"-19\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">MaxPool3x3</text>\n</g>\n<!-- 17954002912330681436 -->\n<g id=\"node22\" class=\"node\">\n<title>17954002912330681436</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"780,-44 696,-44 696,0 780,0 780,-44\"/>\n<text text-anchor=\"start\" x=\"704\" y=\"-28\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; Relu</text>\n<text text-anchor=\"start\" x=\"765\" y=\"-7\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">x3</text>\n</g>\n<!-- /outputs/22&#45;&gt;17954002912330681436 -->\n<g id=\"edge21\" class=\"edge\">\n<title>/outputs/22&#45;&gt;17954002912330681436</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M602.017,-22C626.1037,-22 658.5182,-22 685.6582,-22\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"685.7732,-25.5001 695.7732,-22 685.7732,-18.5001 685.7732,-25.5001\"/>\n<text text-anchor=\"middle\" x=\"649\" y=\"-25\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">256x192x3x3</text>\n</g>\n<!-- /outputs/29 -->\n<g id=\"node3\" class=\"node\">\n<title>/outputs/29</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"944,-40 874,-40 874,-4 944,-4 944,-40\"/>\n<text text-anchor=\"start\" x=\"882\" y=\"-19\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">MaxPool3x3</text>\n</g>\n<!-- /outputs/31 -->\n<g id=\"node5\" class=\"node\">\n<title>/outputs/31</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1168,-67 1038,-67 1038,-31 1168,-31 1168,-67\"/>\n<text text-anchor=\"start\" x=\"1046\" y=\"-46\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">aten::adaptive_avg_pool2d</text>\n</g>\n<!-- /outputs/29&#45;&gt;/outputs/31 -->\n<g id=\"edge1\" class=\"edge\">\n<title>/outputs/29&#45;&gt;/outputs/31</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M944.1319,-26.8895C967.4761,-30.1384 999.0251,-34.5293 1027.8566,-38.5419\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1027.5363,-42.031 1037.9234,-39.9429 1028.5013,-35.0978 1027.5363,-42.031\"/>\n<text text-anchor=\"middle\" x=\"991\" y=\"-40\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">256x256x1x1</text>\n</g>\n<!-- /outputs/30 -->\n<g id=\"node4\" class=\"node\">\n<title>/outputs/30</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"936,-94 882,-94 882,-58 936,-58 936,-94\"/>\n<text text-anchor=\"start\" x=\"891\" y=\"-73\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Constant</text>\n</g>\n<!-- /outputs/30&#45;&gt;/outputs/31 -->\n<g id=\"edge2\" class=\"edge\">\n<title>/outputs/30&#45;&gt;/outputs/31</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M936.384,-72.1888C960.0397,-68.8965 995.3706,-63.9794 1027.4183,-59.5191\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1028.2671,-62.9348 1037.6891,-58.0897 1027.3021,-56.0016 1028.2671,-62.9348\"/>\n</g>\n<!-- /outputs/32 -->\n<g id=\"node6\" class=\"node\">\n<title>/outputs/32</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1316,-97 1262,-97 1262,-61 1316,-61 1316,-97\"/>\n<text text-anchor=\"start\" x=\"1277\" y=\"-76\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Shape</text>\n</g>\n<!-- /outputs/31&#45;&gt;/outputs/32 -->\n<g id=\"edge3\" class=\"edge\">\n<title>/outputs/31&#45;&gt;/outputs/32</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1168.0538,-59.4925C1195.978,-63.9965 1227.6079,-69.0981 1251.5165,-72.9543\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1251.1566,-76.4414 1261.5863,-74.5784 1252.2712,-69.5307 1251.1566,-76.4414\"/>\n<text text-anchor=\"middle\" x=\"1215\" y=\"-73\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">256x256x6x6</text>\n</g>\n<!-- /outputs/39 -->\n<g id=\"node13\" class=\"node\">\n<title>/outputs/39</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1744,-63 1690,-63 1690,-27 1744,-27 1744,-63\"/>\n<text text-anchor=\"start\" x=\"1700\" y=\"-42\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Reshape</text>\n</g>\n<!-- /outputs/31&#45;&gt;/outputs/39 -->\n<g id=\"edge4\" class=\"edge\">\n<title>/outputs/31&#45;&gt;/outputs/39</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1168.3025,-40.6251C1203.9154,-36.7539 1248.8105,-33 1289,-33 1289,-33 1289,-33 1626,-33 1643.6472,-33 1663.0554,-35.2456 1679.3988,-37.7939\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1679.2092,-41.3101 1689.6468,-39.4917 1680.3534,-34.4043 1679.2092,-41.3101\"/>\n<text text-anchor=\"middle\" x=\"1454\" y=\"-36\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">256x256x6x6</text>\n</g>\n<!-- /outputs/34 -->\n<g id=\"node8\" class=\"node\">\n<title>/outputs/34</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1407,-97 1353,-97 1353,-61 1407,-61 1407,-97\"/>\n<text text-anchor=\"start\" x=\"1366\" y=\"-76\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Gather</text>\n</g>\n<!-- /outputs/32&#45;&gt;/outputs/34 -->\n<g id=\"edge5\" class=\"edge\">\n<title>/outputs/32&#45;&gt;/outputs/34</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1316.303,-79C1324.5813,-79 1333.8217,-79 1342.6405,-79\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1342.7163,-82.5001 1352.7163,-79 1342.7162,-75.5001 1342.7163,-82.5001\"/>\n</g>\n<!-- /outputs/33 -->\n<g id=\"node7\" class=\"node\">\n<title>/outputs/33</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1316,-151 1262,-151 1262,-115 1316,-115 1316,-151\"/>\n<text text-anchor=\"start\" x=\"1271\" y=\"-130\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Constant</text>\n</g>\n<!-- /outputs/33&#45;&gt;/outputs/34 -->\n<g id=\"edge6\" class=\"edge\">\n<title>/outputs/33&#45;&gt;/outputs/34</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1316.303,-116.7982C1325.0312,-111.6189 1334.829,-105.8048 1344.0744,-100.3185\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1345.9025,-103.3035 1352.7163,-95.1903 1342.3303,-97.2836 1345.9025,-103.3035\"/>\n</g>\n<!-- /outputs/36 -->\n<g id=\"node10\" class=\"node\">\n<title>/outputs/36</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1562,-97 1501,-97 1501,-61 1562,-61 1562,-97\"/>\n<text text-anchor=\"start\" x=\"1509.5\" y=\"-76\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Unsqueeze</text>\n</g>\n<!-- /outputs/34&#45;&gt;/outputs/36 -->\n<g id=\"edge7\" class=\"edge\">\n<title>/outputs/34&#45;&gt;/outputs/36</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1407.0852,-79C1430.3025,-79 1464.097,-79 1490.6073,-79\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1490.7376,-82.5001 1500.7376,-79 1490.7376,-75.5001 1490.7376,-82.5001\"/>\n</g>\n<!-- /outputs/35 -->\n<g id=\"node9\" class=\"node\">\n<title>/outputs/35</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1407,-151 1353,-151 1353,-115 1407,-115 1407,-151\"/>\n<text text-anchor=\"start\" x=\"1362\" y=\"-130\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Constant</text>\n</g>\n<!-- /outputs/37 -->\n<g id=\"node11\" class=\"node\">\n<title>/outputs/37</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1562,-151 1501,-151 1501,-115 1562,-115 1562,-151\"/>\n<text text-anchor=\"start\" x=\"1509.5\" y=\"-130\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Unsqueeze</text>\n</g>\n<!-- /outputs/35&#45;&gt;/outputs/37 -->\n<g id=\"edge8\" class=\"edge\">\n<title>/outputs/35&#45;&gt;/outputs/37</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1407.0852,-133C1430.3025,-133 1464.097,-133 1490.6073,-133\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1490.7376,-136.5001 1500.7376,-133 1490.7376,-129.5001 1490.7376,-136.5001\"/>\n</g>\n<!-- /outputs/38 -->\n<g id=\"node12\" class=\"node\">\n<title>/outputs/38</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1653,-97 1599,-97 1599,-61 1653,-61 1653,-97\"/>\n<text text-anchor=\"start\" x=\"1611\" y=\"-76\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Concat</text>\n</g>\n<!-- /outputs/36&#45;&gt;/outputs/38 -->\n<g id=\"edge9\" class=\"edge\">\n<title>/outputs/36&#45;&gt;/outputs/38</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1562.1814,-79C1570.7033,-79 1580.0485,-79 1588.8957,-79\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1588.9735,-82.5001 1598.9735,-79 1588.9734,-75.5001 1588.9735,-82.5001\"/>\n</g>\n<!-- /outputs/37&#45;&gt;/outputs/38 -->\n<g id=\"edge10\" class=\"edge\">\n<title>/outputs/37&#45;&gt;/outputs/38</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1562.1814,-115.4677C1571.0738,-110.3864 1580.8627,-104.7927 1590.0467,-99.5447\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1592.0275,-102.444 1598.9735,-94.4437 1588.5545,-96.3663 1592.0275,-102.444\"/>\n</g>\n<!-- /outputs/38&#45;&gt;/outputs/39 -->\n<g id=\"edge11\" class=\"edge\">\n<title>/outputs/38&#45;&gt;/outputs/39</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1653.303,-68.7989C1661.7612,-65.6387 1671.224,-62.1031 1680.215,-58.7438\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1681.5738,-61.9726 1689.7163,-55.1939 1679.1237,-55.4153 1681.5738,-61.9726\"/>\n</g>\n<!-- /outputs/40/41 -->\n<g id=\"node14\" class=\"node\">\n<title>/outputs/40/41</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1877,-63 1823,-63 1823,-27 1877,-27 1877,-63\"/>\n<text text-anchor=\"start\" x=\"1833\" y=\"-42\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Dropout</text>\n</g>\n<!-- /outputs/39&#45;&gt;/outputs/40/41 -->\n<g id=\"edge12\" class=\"edge\">\n<title>/outputs/39&#45;&gt;/outputs/40/41</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1744.2355,-45C1763.9593,-45 1790.8628,-45 1812.5994,-45\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1812.801,-48.5001 1822.8009,-45 1812.8009,-41.5001 1812.801,-48.5001\"/>\n<text text-anchor=\"middle\" x=\"1783.5\" y=\"-48\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">256x9216</text>\n</g>\n<!-- 3295757833782713099 -->\n<g id=\"node20\" class=\"node\">\n<title>3295757833782713099</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"2028,-63 1956,-63 1956,-27 2028,-27 2028,-63\"/>\n<text text-anchor=\"start\" x=\"1964\" y=\"-42\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Linear &gt; Relu</text>\n</g>\n<!-- /outputs/40/41&#45;&gt;3295757833782713099 -->\n<g id=\"edge17\" class=\"edge\">\n<title>/outputs/40/41&#45;&gt;3295757833782713099</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1877.0434,-45C1896.4426,-45 1923.0598,-45 1945.7969,-45\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1945.9308,-48.5001 1955.9307,-45 1945.9307,-41.5001 1945.9308,-48.5001\"/>\n<text text-anchor=\"middle\" x=\"1916.5\" y=\"-48\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">256x9216</text>\n</g>\n<!-- /outputs/44/45 -->\n<g id=\"node15\" class=\"node\">\n<title>/outputs/44/45</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"2161,-63 2107,-63 2107,-27 2161,-27 2161,-63\"/>\n<text text-anchor=\"start\" x=\"2117\" y=\"-42\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Dropout</text>\n</g>\n<!-- 3249489531966621551 -->\n<g id=\"node21\" class=\"node\">\n<title>3249489531966621551</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"2312,-63 2240,-63 2240,-27 2312,-27 2312,-63\"/>\n<text text-anchor=\"start\" x=\"2248\" y=\"-42\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Linear &gt; Relu</text>\n</g>\n<!-- /outputs/44/45&#45;&gt;3249489531966621551 -->\n<g id=\"edge19\" class=\"edge\">\n<title>/outputs/44/45&#45;&gt;3249489531966621551</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M2161.0434,-45C2180.4426,-45 2207.0598,-45 2229.7969,-45\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2229.9308,-48.5001 2239.9307,-45 2229.9307,-41.5001 2229.9308,-48.5001\"/>\n<text text-anchor=\"middle\" x=\"2200.5\" y=\"-48\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">256x4096</text>\n</g>\n<!-- /outputs/48 -->\n<g id=\"node16\" class=\"node\">\n<title>/outputs/48</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"2445,-63 2391,-63 2391,-27 2445,-27 2445,-63\"/>\n<text text-anchor=\"start\" x=\"2405\" y=\"-42\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Linear</text>\n</g>\n<!-- /outputs/49 -->\n<g id=\"node17\" class=\"node\">\n<title>/outputs/49</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"2567,-63 2513,-63 2513,-27 2567,-27 2567,-63\"/>\n<text text-anchor=\"start\" x=\"2523\" y=\"-42\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Softmax</text>\n</g>\n<!-- /outputs/48&#45;&gt;/outputs/49 -->\n<g id=\"edge13\" class=\"edge\">\n<title>/outputs/48&#45;&gt;/outputs/49</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M2445.0758,-45C2462.0553,-45 2484.1767,-45 2502.7924,-45\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2503,-48.5001 2512.9999,-45 2502.9999,-41.5001 2503,-48.5001\"/>\n<text text-anchor=\"middle\" x=\"2479\" y=\"-48\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">256x10</text>\n</g>\n<!-- 16847199909073694331 -->\n<g id=\"node18\" class=\"node\">\n<title>16847199909073694331</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"95,-40 0,-40 0,-4 95,-4 95,-40\"/>\n<text text-anchor=\"start\" x=\"8.5\" y=\"-19\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv11x11 &gt; Relu</text>\n</g>\n<!-- 16847199909073694331&#45;&gt;/outputs/19 -->\n<g id=\"edge14\" class=\"edge\">\n<title>16847199909073694331&#45;&gt;/outputs/19</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M95.4497,-22C123.075,-22 157.5932,-22 184.7502,-22\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"184.7683,-25.5001 194.7682,-22 184.7682,-18.5001 184.7683,-25.5001\"/>\n<text text-anchor=\"middle\" x=\"145\" y=\"-25\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">256x64x15x15</text>\n</g>\n<!-- 4813633701811250313&#45;&gt;/outputs/22 -->\n<g id=\"edge16\" class=\"edge\">\n<title>4813633701811250313&#45;&gt;/outputs/22</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M438.2697,-22C463.5651,-22 495.7113,-22 521.5519,-22\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"521.8392,-25.5001 531.8391,-22 521.8391,-18.5001 521.8392,-25.5001\"/>\n<text text-anchor=\"middle\" x=\"485\" y=\"-25\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">256x192x7x7</text>\n</g>\n<!-- 3295757833782713099&#45;&gt;/outputs/44/45 -->\n<g id=\"edge18\" class=\"edge\">\n<title>3295757833782713099&#45;&gt;/outputs/44/45</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M2028.2,-45C2049.1476,-45 2075.4645,-45 2096.6106,-45\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2096.8349,-48.5001 2106.8349,-45 2096.8348,-41.5001 2096.8349,-48.5001\"/>\n<text text-anchor=\"middle\" x=\"2067.5\" y=\"-48\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">256x4096</text>\n</g>\n<!-- 3249489531966621551&#45;&gt;/outputs/48 -->\n<g id=\"edge20\" class=\"edge\">\n<title>3249489531966621551&#45;&gt;/outputs/48</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M2312.2,-45C2333.1476,-45 2359.4645,-45 2380.6106,-45\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2380.8349,-48.5001 2390.8349,-45 2380.8348,-41.5001 2380.8349,-48.5001\"/>\n<text text-anchor=\"middle\" x=\"2351.5\" y=\"-48\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">256x4096</text>\n</g>\n<!-- 17954002912330681436&#45;&gt;/outputs/29 -->\n<g id=\"edge22\" class=\"edge\">\n<title>17954002912330681436&#45;&gt;/outputs/29</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M780.2697,-22C805.5651,-22 837.7113,-22 863.5519,-22\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"863.8392,-25.5001 873.8391,-22 863.8391,-18.5001 863.8392,-25.5001\"/>\n<text text-anchor=\"middle\" x=\"827\" y=\"-25\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">256x256x3x3</text>\n</g>\n</g>\n</svg>\n"
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "RAodboScj5w6"
      },
      "source": [
        "## Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Dzh3ROmRj5w7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "7d68418a-6421-4f2d-ac49-60d38201be8a"
      },
      "source": [
        "def compute_acc(model, data_loader, device):\n",
        "    correct_pred, num_examples = 0, 0\n",
        "    model.eval()\n",
        "    for i, (features, targets) in enumerate(data_loader):\n",
        "            \n",
        "        features = features.to(device)\n",
        "        targets = targets.to(device)\n",
        "\n",
        "        logits, probas = model(features)\n",
        "        _, predicted_labels = torch.max(probas, 1)\n",
        "        num_examples += targets.size(0)\n",
        "        assert predicted_labels.size() == targets.size()\n",
        "        correct_pred += (predicted_labels == targets).sum()\n",
        "    return correct_pred.float()/num_examples * 100\n",
        "    \n",
        "\n",
        "start_time = time.time()\n",
        "\n",
        "cost_list = []\n",
        "train_acc_list, valid_acc_list = [], []\n",
        "\n",
        "\n",
        "for epoch in range(NUM_EPOCHS):\n",
        "    \n",
        "    model.train()\n",
        "    for batch_idx, (features, targets) in enumerate(train_loader):\n",
        "        \n",
        "        features = features.to(DEVICE)\n",
        "        targets = targets.to(DEVICE)\n",
        "            \n",
        "        ### FORWARD AND BACK PROP\n",
        "        logits, probas = model(features)\n",
        "        cost = F.cross_entropy(logits, targets)\n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        cost.backward()\n",
        "        \n",
        "        ### UPDATE MODEL PARAMETERS\n",
        "        optimizer.step()\n",
        "        \n",
        "        #################################################\n",
        "        ### CODE ONLY FOR LOGGING BEYOND THIS POINT\n",
        "        ################################################\n",
        "        cost_list.append(cost.item())\n",
        "        if not batch_idx % 150:\n",
        "            print (f'Epoch: {epoch+1:03d}/{NUM_EPOCHS:03d} | '\n",
        "                   f'Batch {batch_idx:03d}/{len(train_loader):03d} |' \n",
        "                   f' Cost: {cost:.4f}')\n",
        "\n",
        "        \n",
        "\n",
        "    model.eval()\n",
        "    with torch.set_grad_enabled(False): # save memory during inference\n",
        "        \n",
        "        train_acc = compute_acc(model, train_loader, device=DEVICE)\n",
        "        valid_acc = compute_acc(model, valid_loader, device=DEVICE)\n",
        "        \n",
        "        print(f'Epoch: {epoch+1:03d}/{NUM_EPOCHS:03d}\\n'\n",
        "              f'Train ACC: {train_acc:.2f} | Validation ACC: {valid_acc:.2f}')\n",
        "        \n",
        "        train_acc_list.append(train_acc)\n",
        "        valid_acc_list.append(valid_acc)\n",
        "        \n",
        "    elapsed = (time.time() - start_time)/60\n",
        "    print(f'Time elapsed: {elapsed:.2f} min')\n",
        "  \n",
        "elapsed = (time.time() - start_time)/60\n",
        "print(f'Total Training Time: {elapsed:.2f} min')"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 001/040 | Batch 000/188 | Cost: 2.3029\n",
            "Epoch: 001/040 | Batch 150/188 | Cost: 1.7611\n",
            "Epoch: 001/040\n",
            "Train ACC: 32.21 | Validation ACC: 32.45\n",
            "Time elapsed: 0.43 min\n",
            "Epoch: 002/040 | Batch 000/188 | Cost: 1.6974\n",
            "Epoch: 002/040 | Batch 150/188 | Cost: 1.6413\n",
            "Epoch: 002/040\n",
            "Train ACC: 41.61 | Validation ACC: 39.95\n",
            "Time elapsed: 0.86 min\n",
            "Epoch: 003/040 | Batch 000/188 | Cost: 1.5619\n",
            "Epoch: 003/040 | Batch 150/188 | Cost: 1.4806\n",
            "Epoch: 003/040\n",
            "Train ACC: 52.60 | Validation ACC: 52.85\n",
            "Time elapsed: 1.30 min\n",
            "Epoch: 004/040 | Batch 000/188 | Cost: 1.3688\n",
            "Epoch: 004/040 | Batch 150/188 | Cost: 1.2667\n",
            "Epoch: 004/040\n",
            "Train ACC: 58.24 | Validation ACC: 58.40\n",
            "Time elapsed: 1.73 min\n",
            "Epoch: 005/040 | Batch 000/188 | Cost: 1.0639\n",
            "Epoch: 005/040 | Batch 150/188 | Cost: 1.1401\n",
            "Epoch: 005/040\n",
            "Train ACC: 60.57 | Validation ACC: 59.40\n",
            "Time elapsed: 2.16 min\n",
            "Epoch: 006/040 | Batch 000/188 | Cost: 1.0465\n",
            "Epoch: 006/040 | Batch 150/188 | Cost: 1.0986\n",
            "Epoch: 006/040\n",
            "Train ACC: 63.06 | Validation ACC: 61.80\n",
            "Time elapsed: 2.60 min\n",
            "Epoch: 007/040 | Batch 000/188 | Cost: 0.9800\n",
            "Epoch: 007/040 | Batch 150/188 | Cost: 1.0789\n",
            "Epoch: 007/040\n",
            "Train ACC: 66.27 | Validation ACC: 64.25\n",
            "Time elapsed: 3.04 min\n",
            "Epoch: 008/040 | Batch 000/188 | Cost: 1.0372\n",
            "Epoch: 008/040 | Batch 150/188 | Cost: 0.9276\n",
            "Epoch: 008/040\n",
            "Train ACC: 64.99 | Validation ACC: 64.55\n",
            "Time elapsed: 3.47 min\n",
            "Epoch: 009/040 | Batch 000/188 | Cost: 1.0141\n",
            "Epoch: 009/040 | Batch 150/188 | Cost: 0.8099\n",
            "Epoch: 009/040\n",
            "Train ACC: 70.65 | Validation ACC: 66.50\n",
            "Time elapsed: 3.91 min\n",
            "Epoch: 010/040 | Batch 000/188 | Cost: 0.7600\n",
            "Epoch: 010/040 | Batch 150/188 | Cost: 0.7787\n",
            "Epoch: 010/040\n",
            "Train ACC: 71.02 | Validation ACC: 65.85\n",
            "Time elapsed: 4.35 min\n",
            "Epoch: 011/040 | Batch 000/188 | Cost: 0.8431\n",
            "Epoch: 011/040 | Batch 150/188 | Cost: 0.7991\n",
            "Epoch: 011/040\n",
            "Train ACC: 70.70 | Validation ACC: 65.40\n",
            "Time elapsed: 4.78 min\n",
            "Epoch: 012/040 | Batch 000/188 | Cost: 0.7612\n",
            "Epoch: 012/040 | Batch 150/188 | Cost: 0.9599\n",
            "Epoch: 012/040\n",
            "Train ACC: 76.07 | Validation ACC: 69.00\n",
            "Time elapsed: 5.22 min\n",
            "Epoch: 013/040 | Batch 000/188 | Cost: 0.6079\n",
            "Epoch: 013/040 | Batch 150/188 | Cost: 0.6792\n",
            "Epoch: 013/040\n",
            "Train ACC: 78.28 | Validation ACC: 70.15\n",
            "Time elapsed: 5.66 min\n",
            "Epoch: 014/040 | Batch 000/188 | Cost: 0.6128\n",
            "Epoch: 014/040 | Batch 150/188 | Cost: 0.6522\n",
            "Epoch: 014/040\n",
            "Train ACC: 77.44 | Validation ACC: 69.30\n",
            "Time elapsed: 6.09 min\n",
            "Epoch: 015/040 | Batch 000/188 | Cost: 0.5693\n",
            "Epoch: 015/040 | Batch 150/188 | Cost: 0.7446\n",
            "Epoch: 015/040\n",
            "Train ACC: 77.25 | Validation ACC: 70.55\n",
            "Time elapsed: 6.54 min\n",
            "Epoch: 016/040 | Batch 000/188 | Cost: 0.8091\n",
            "Epoch: 016/040 | Batch 150/188 | Cost: 0.5905\n",
            "Epoch: 016/040\n",
            "Train ACC: 79.34 | Validation ACC: 70.65\n",
            "Time elapsed: 6.98 min\n",
            "Epoch: 017/040 | Batch 000/188 | Cost: 0.6292\n",
            "Epoch: 017/040 | Batch 150/188 | Cost: 0.5726\n",
            "Epoch: 017/040\n",
            "Train ACC: 82.61 | Validation ACC: 71.85\n",
            "Time elapsed: 7.42 min\n",
            "Epoch: 018/040 | Batch 000/188 | Cost: 0.5483\n",
            "Epoch: 018/040 | Batch 150/188 | Cost: 0.4530\n",
            "Epoch: 018/040\n",
            "Train ACC: 83.22 | Validation ACC: 72.15\n",
            "Time elapsed: 7.86 min\n",
            "Epoch: 019/040 | Batch 000/188 | Cost: 0.5313\n",
            "Epoch: 019/040 | Batch 150/188 | Cost: 0.4551\n",
            "Epoch: 019/040\n",
            "Train ACC: 85.05 | Validation ACC: 73.05\n",
            "Time elapsed: 8.29 min\n",
            "Epoch: 020/040 | Batch 000/188 | Cost: 0.4801\n",
            "Epoch: 020/040 | Batch 150/188 | Cost: 0.3646\n",
            "Epoch: 020/040\n",
            "Train ACC: 84.97 | Validation ACC: 72.85\n",
            "Time elapsed: 8.73 min\n",
            "Epoch: 021/040 | Batch 000/188 | Cost: 0.4229\n",
            "Epoch: 021/040 | Batch 150/188 | Cost: 0.4474\n",
            "Epoch: 021/040\n",
            "Train ACC: 87.07 | Validation ACC: 72.80\n",
            "Time elapsed: 9.18 min\n",
            "Epoch: 022/040 | Batch 000/188 | Cost: 0.3776\n",
            "Epoch: 022/040 | Batch 150/188 | Cost: 0.4258\n",
            "Epoch: 022/040\n",
            "Train ACC: 89.23 | Validation ACC: 72.80\n",
            "Time elapsed: 9.63 min\n",
            "Epoch: 023/040 | Batch 000/188 | Cost: 0.2967\n",
            "Epoch: 023/040 | Batch 150/188 | Cost: 0.3287\n",
            "Epoch: 023/040\n",
            "Train ACC: 88.45 | Validation ACC: 72.50\n",
            "Time elapsed: 10.07 min\n",
            "Epoch: 024/040 | Batch 000/188 | Cost: 0.3499\n",
            "Epoch: 024/040 | Batch 150/188 | Cost: 0.4624\n",
            "Epoch: 024/040\n",
            "Train ACC: 89.43 | Validation ACC: 73.95\n",
            "Time elapsed: 10.51 min\n",
            "Epoch: 025/040 | Batch 000/188 | Cost: 0.2493\n",
            "Epoch: 025/040 | Batch 150/188 | Cost: 0.3115\n",
            "Epoch: 025/040\n",
            "Train ACC: 91.94 | Validation ACC: 73.10\n",
            "Time elapsed: 10.96 min\n",
            "Epoch: 026/040 | Batch 000/188 | Cost: 0.2510\n",
            "Epoch: 026/040 | Batch 150/188 | Cost: 0.3057\n",
            "Epoch: 026/040\n",
            "Train ACC: 89.90 | Validation ACC: 72.80\n",
            "Time elapsed: 11.40 min\n",
            "Epoch: 027/040 | Batch 000/188 | Cost: 0.3399\n",
            "Epoch: 027/040 | Batch 150/188 | Cost: 0.2944\n",
            "Epoch: 027/040\n",
            "Train ACC: 92.43 | Validation ACC: 73.80\n",
            "Time elapsed: 11.85 min\n",
            "Epoch: 028/040 | Batch 000/188 | Cost: 0.2326\n",
            "Epoch: 028/040 | Batch 150/188 | Cost: 0.2142\n",
            "Epoch: 028/040\n",
            "Train ACC: 93.18 | Validation ACC: 73.35\n",
            "Time elapsed: 12.30 min\n",
            "Epoch: 029/040 | Batch 000/188 | Cost: 0.1595\n",
            "Epoch: 029/040 | Batch 150/188 | Cost: 0.1496\n",
            "Epoch: 029/040\n",
            "Train ACC: 92.75 | Validation ACC: 72.45\n",
            "Time elapsed: 12.74 min\n",
            "Epoch: 030/040 | Batch 000/188 | Cost: 0.1536\n",
            "Epoch: 030/040 | Batch 150/188 | Cost: 0.2648\n",
            "Epoch: 030/040\n",
            "Train ACC: 94.60 | Validation ACC: 74.65\n",
            "Time elapsed: 13.19 min\n",
            "Epoch: 031/040 | Batch 000/188 | Cost: 0.1446\n",
            "Epoch: 031/040 | Batch 150/188 | Cost: 0.2176\n",
            "Epoch: 031/040\n",
            "Train ACC: 95.20 | Validation ACC: 72.70\n",
            "Time elapsed: 13.63 min\n",
            "Epoch: 032/040 | Batch 000/188 | Cost: 0.1867\n",
            "Epoch: 032/040 | Batch 150/188 | Cost: 0.2059\n",
            "Epoch: 032/040\n",
            "Train ACC: 94.85 | Validation ACC: 72.20\n",
            "Time elapsed: 14.07 min\n",
            "Epoch: 033/040 | Batch 000/188 | Cost: 0.1519\n",
            "Epoch: 033/040 | Batch 150/188 | Cost: 0.1610\n",
            "Epoch: 033/040\n",
            "Train ACC: 94.45 | Validation ACC: 73.60\n",
            "Time elapsed: 14.52 min\n",
            "Epoch: 034/040 | Batch 000/188 | Cost: 0.1914\n",
            "Epoch: 034/040 | Batch 150/188 | Cost: 0.1273\n",
            "Epoch: 034/040\n",
            "Train ACC: 95.95 | Validation ACC: 74.55\n",
            "Time elapsed: 14.96 min\n",
            "Epoch: 035/040 | Batch 000/188 | Cost: 0.1706\n",
            "Epoch: 035/040 | Batch 150/188 | Cost: 0.2138\n",
            "Epoch: 035/040\n",
            "Train ACC: 95.38 | Validation ACC: 73.30\n",
            "Time elapsed: 15.40 min\n",
            "Epoch: 036/040 | Batch 000/188 | Cost: 0.1651\n",
            "Epoch: 036/040 | Batch 150/188 | Cost: 0.0744\n",
            "Epoch: 036/040\n",
            "Train ACC: 95.57 | Validation ACC: 73.40\n",
            "Time elapsed: 15.85 min\n",
            "Epoch: 037/040 | Batch 000/188 | Cost: 0.1129\n",
            "Epoch: 037/040 | Batch 150/188 | Cost: 0.1390\n",
            "Epoch: 037/040\n",
            "Train ACC: 95.49 | Validation ACC: 71.90\n",
            "Time elapsed: 16.29 min\n",
            "Epoch: 038/040 | Batch 000/188 | Cost: 0.1277\n",
            "Epoch: 038/040 | Batch 150/188 | Cost: 0.1193\n",
            "Epoch: 038/040\n",
            "Train ACC: 96.37 | Validation ACC: 73.45\n",
            "Time elapsed: 16.74 min\n",
            "Epoch: 039/040 | Batch 000/188 | Cost: 0.1258\n",
            "Epoch: 039/040 | Batch 150/188 | Cost: 0.0917\n",
            "Epoch: 039/040\n",
            "Train ACC: 96.66 | Validation ACC: 74.20\n",
            "Time elapsed: 17.18 min\n",
            "Epoch: 040/040 | Batch 000/188 | Cost: 0.1312\n",
            "Epoch: 040/040 | Batch 150/188 | Cost: 0.1471\n",
            "Epoch: 040/040\n",
            "Train ACC: 96.40 | Validation ACC: 73.70\n",
            "Time elapsed: 17.62 min\n",
            "Total Training Time: 17.62 min\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zt2G8n8ld4eX",
        "colab_type": "text"
      },
      "source": [
        "## Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B9rmVnmbd4eY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_pCH64Qud4ea",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "outputId": "f222b059-b833-4936-ceda-c8239a65705c"
      },
      "source": [
        "plt.plot(cost_list, label='Minibatch cost')\n",
        "plt.plot(np.convolve(cost_list, \n",
        "                     np.ones(200,)/200, mode='valid'), \n",
        "         label='Running average')\n",
        "\n",
        "plt.ylabel('Cross Entropy')\n",
        "plt.xlabel('Iteration')\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3xUxRbA8d9sDaGX0EvoLTQpUlS6ICAiCCgoRREbYnkWEBTFhoWnoKhPbKiICCqiIEgRAamhh94CJLTQS0iyZd4fm2zaZrMpmwT2fD+fvLd779x7T6Lu2Xtn5ozSWiOEECJwGfI7ACGEEPlLEoEQQgQ4SQRCCBHgJBEIIUSAk0QghBABzpTfAWRVmTJldGhoaH6HIYQQ15VNmzad0VqHeNp33SWC0NBQwsPD8zsMIYS4riiljmS0Tx4NCSFEgJNEIIQQAU4SgRBCBLjrro9ACJEzNpuNqKgo4uLi8jsU4QdBQUFUrlwZs9ns8zGSCIQIMFFRURQtWpTQ0FCUUvkdjshFWmvOnj1LVFQU1atX9/k4eTQkRICJi4ujdOnSkgRuQEopSpcuneW7PUkEQgQgSQI3ruz8sw2oRLBs9ymiL1zL7zCEEKJACZhEcOD0ZR6aEU67ScvzOxQhAp5Sivvvv9/93m63ExISQq9evQCYP38+kyZN8nqO48ePc8899wDwzTffMGrUqCzF8NZbb2XaZtiwYcydOzdL582OrVu3snDhQr9fJyMBkwj+3hOT3yEIIRIVLlyYiIgIrl1z3aEvWbKESpUquff37t2bMWPGeD1HxYoVc/Qh7UsiyCuSCPLIiFt970EXQvhfjx49WLBgAQCzZs3ivvvuc+9L+Q1/2LBhjB49mrZt21KjRg33h39kZCRhYWHuY44dO0aHDh2oXbs2r732mnt7nz59aN68OQ0bNuTzzz8HYMyYMVy7do2mTZsyePBgAL799lsaN25MkyZNeOCBB9zHr1y5Mt210/J0bGRkJJ06daJx48Z07tyZo0ePAjBnzhzCwsJo0qQJt912GwkJCbzyyivMnj2bpk2bMnv27Jz9YbMhYIaPpuxAmfzXXno1rkjd8kXzMSIh8t9rv+9k1/FLuXrOBhWLMeHOhpm2u/fee5k4cSK9evVi+/btPPjgg6xatcpj2xMnTrB69Wr27NlD79693Y+EUtqwYQMREREEBwfTsmVLevbsSYsWLfjqq68oVaoU165do2XLlvTr149Jkybx8ccfs3XrVgB27tzJG2+8wZo1ayhTpgznzp3z+doZHfvkk08ydOhQhg4dyldffcXo0aOZN28eEydOZPHixVSqVIkLFy5gsViYOHEi4eHhfPzxxz7/nXNTwNwRAIzqWAuAj5YfoP9na/I5GiECW+PGjYmMjGTWrFn06NHDa9s+ffpgMBho0KABp06d8tima9eulC5dmkKFCtG3b19Wr14NwNSpU2nSpAmtW7fm2LFj7N+/P92xy5cvp3///pQpUwaAUqVK+XztjI5du3YtgwYNAuCBBx5wx9OuXTuGDRvG9OnTcTgcXn/vvBIwdwQAQebkvOdw6nyMRIiCwZdv7v7Uu3dvnnvuOVasWMHZs2czbGe1Wt2vtfb8327aYZNKKVasWMHSpUtZu3YtwcHBdOjQIctj7H25dlZ89tlnrF+/ngULFtC8eXM2bdqU43PmVEDdEVhNRvdrGUctRP578MEHmTBhAo0aNcrxuZYsWcK5c+e4du0a8+bNo127dly8eJGSJUsSHBzMnj17WLdunbu92WzGZrMB0KlTJ+bMmeNORikfDWUmo2Pbtm3Ljz/+CMDMmTO59dZbATh48CA333wzEydOJCQkhGPHjlG0aFEuX76c479BdgVUIkh5RyCEyH+VK1dm9OjRuXKuVq1a0a9fPxo3bky/fv1o0aIF3bt3x263U79+fcaMGUPr1q3d7UeOHEnjxo0ZPHgwDRs2ZNy4cbRv354mTZrw7LPP+nzdjI796KOP+Prrr2ncuDHfffcdU6ZMAeD555+nUaNGhIWF0bZtW5o0aULHjh3ZtWtXvnUWq9y41clLLVq00NldmOan8GO8MHc7AEWtJna81i03QxPiurB7927q16+f32EIP/L0z1gptUlr3cJT+4D6ihxsMWbeSAghAkxAJYLClhR949JFIIQQQIAlArkjEEKI9AIqERS2Jt8RyA2BEEK4BFQikDsCIYRIL6ASgdUsiUAIIdIKqERgTDGJTCaUCZF/jEYjTZs2JSwsjDvvvJMLFy7k+jU+++wzvv3221w/740ooBKBIaB+WyEKrkKFCrF161YiIiIoVaoU06ZNy/VrPProowwZMiTXz5tb7HZ7fofgFlAfjYZUdwT5GIgQwq1NmzZER0cD0KFDB5ImjJ45c4bQ0FDAVZa6b9++dO/endq1a/PCCy+4jy9SpAjjxo1zF5ZLKgz36quv8v7777vP++KLL9KqVSvq1KnjrnIaGxvLgAEDaNCgAXfffTc333wzniasTpw4kZYtWxIWFsbIkSPRWrNnzx5atWrlbhMZGekulbFp0ybat29P8+bN6datGydOnHDH8fTTT9OiRQumTJnC77//zs0330yzZs3o0qWLO/aYmBi6du1Kw4YNGTFiBNWqVePMmTMAfP/997Rq1YqmTZvyyCOP5ErhuoAqOief/UKk8ecYOLkjd89ZvhHc4X11sSQOh4Nly5bx0EMPZdp269atbNmyBavVSt26dXnyySepUqUKV69epXXr1rz55pu88MILTJ8+nfHjx6c73m63s2HDBhYuXMhrr73G0qVL+eSTTyhZsiS7du0iIiKCpk2berz2qFGjeOWVVwBXJdE//viDO++8k4SEBA4fPkz16tWZPXs2AwcOxGaz8eSTT/Lbb78REhLC7NmzGTduHF999RUACQkJ7mRz/vx51q1bh1KKL774gnfffZfJkyfz2muv0alTJ8aOHcuiRYv48ssvAdeM4dmzZ/Pvv/9iNpt5/PHHmTlzZo7vfAIrEaS4DbgQa8vHSIQIbEmLwkRHR1O/fn26du2a6TGdO3emePHiADRo0IAjR45QpUoVLBaLe4nL5s2bs2TJEo/H9+3b190mMjISgNWrV/PUU08BEBYWRuPGjT0e+/fff/Puu+8SGxvLuXPnaNiwIXfeeScDBgxg9uzZjBkzhtmzZzN79mz27t1LRESE+3dyOBxUqFDBfa6BAwe6X0dFRTFw4EBOnDhBQkIC1atXd8f166+/AtC9e3dKliwJwLJly9i0aRMtW7Z0/x3Lli2b6d8uMwGVCIQQafj4zT23JfURxMbG0q1bN6ZNm8bo0aMxmUw4nU6AdOWiU5aDNhqN7mfsZrPZ/SUv5fa0ko731saTuLg4Hn/8ccLDw6lSpQqvvvqqO7aBAwfSv39/+vbti1KK2rVrs2PHDho2bMjatWs9nq9w4cLu108++STPPvssvXv3ZsWKFbz66qteY9FaM3ToUN5++22f4/dFQPURmIzycEiIgiQ4OJipU6cyefJk7HY7oaGh7vr8ebFofLt27fjpp58A2LVrFzt2pH9MlvShX6ZMGa5cuZIqrpo1a2I0Gnn99dfd3/Tr1q1LTEyMOxHYbDZ27tzp8foXL150r9U8Y8YMj3H99ddfnD9/HnDdFc2dO5fTp08DrpLXR44cyf4fIFFAJYJiQeb8DkEIkUazZs1o3Lgxs2bN4rnnnuPTTz+lWbNm7s5Rf3r88ceJiYmhQYMGjB8/noYNG7ofPyUpUaIEDz/8MGFhYXTr1s39WCbJwIED+f777xkwYAAAFouFuXPn8uKLL9KkSROaNm3KmjWeV0R89dVX6d+/P82bN3evcAYwYcIE/vrrL8LCwpgzZw7ly5enaNGiNGjQgDfeeIPbb7+dxo0b07VrV3dHdE4EVBlqgNAxC9yvIyf1zI2QhLiuSBnqZA6HA5vNRlBQEAcPHqRLly7s3bsXi8WSr3HFx8djNBoxmUysXbuWxx57zL2+si+yWobab30ESqkqwLdAOUADn2utp6Rpo4ApQA8gFhimtd7sr5iEECKl2NhYOnbsiM1mQ2vNJ598ku9JAODo0aMMGDAAp9OJxWJh+vTpfr2ePzuL7cB/tNablVJFgU1KqSVa610p2twB1E78uRn4NPH/hRDC74oWLepx3kB+q127Nlu2bMmz6/mtj0BrfSLp273W+jKwG6iUptldwLfaZR1QQilVgTxid7hGJ8QmFJwZfkLkhevtkbDwXXb+2eZJZ7FSKhRoBqxPs6sScCzF+yjSJwu/qTXuT/7ccYIGrywmIvpiXl1WiHwVFBTE2bNnJRncgLTWnD17lqCgoCwd5/d5BEqpIsDPwNNa60vZPMdIYCRA1apVczE6+GdfDAA7oi8SVql4Jq2FuP5VrlyZqKgoYmJi8jsU4QdBQUFUrlw5S8f4NREopcy4ksBMrfUvHppEA1VSvK+cuC0VrfXnwOfgGjWUk5geua0G/1t5KCenEOK6Zjab3TNYhQA/PhpKHBH0JbBba/3fDJrNB4Yol9bARa11zgfFemEwpJ5UJnfHQohA5887gnbAA8AOpVTSANiXgKoAWuvPgIW4ho4ewDV8dLgf4wFSr0kghBDCj4lAa72aTAp+aldv1RP+isGTtHcEQggR6AKqxARARnnAljiUVAghAk3gJIL9S2HazRRNSF2/xJnYSfDKb56LQgkhxI0ucBKByQoxewi5diDV5jmbovIpICGEKBgCJxGUawhAyNUDmTQUQojAEjiJILgUFK1As6Dj+R2JEEIUKIGTCADKNSTo7B5mjpC6dkIIkSSwEkHZBnBmL2UKBdavLYQQ3gTWJ2K5MHAkUNd8Kr8jEUKIAiPAEkED1/+f8jxU9MjZq3kYjBBCFAyBlQjK1AGDKcNEEHX+Wh4HJIQQ+S+wEoHJCqVrw+ldHnc7nFKBTggReAIrEYBrPsGpXdxau0y6XU4pRSqECEABmAgawMWjdAq1ptt1KU6WrBRCBJ4ATARhABS/tC/drtGzthA6ZkFeRySEEPkq8BJBlVaAov61LRk2uRRny7t4hBAinwVeIihUEso3op4tIsMmWipSCyECSOAlAoBqbVHHNmLGc5/AzhMX8zggIYTIPwGbCLBf482bPSeCQdPX53FAQgiRfwIzEVRtC0CvYgczbKJlKKkQIkAEZiIoEgLlG2M6uCTDJmN+3oFdlq8UQgSAwEwEAHV7YDm+kVJc8rh7dvgxJv25J4+DEkKIvBfAieAOQNPRsDXDJn/vPZ138QghRD4J3ERQoQkUrUi/ItszbHIwRqqRCiFufIGbCJSCut1pq7dRtWjGf4YdUclDSbXWfLBkH9EXpEqpEOLGEbiJAKBuD7BdZek9Gf8Z7vx4tfv1/tNXmLJsP499vykvohNCiDwR2Ikg9FawFMG08xfvzcYsIM7mcJepjrfJaCIhxI0jsBOBOQiaDkJFzCWE816bfrA0uUidUv4OTAgh8k5gJwKAFg+hnHZ6Gr3PJv7fP4c4GHMlj4ISQoi8I4mgbD0oF0Zv45pMm476IeOKpUIIcb2SRADQqD83GQ5QW0XldyRCCJHnJBEA3DQETIUYblyU35EIIUSek0QAEFwKGt5Nb+MagonLtLlK7C3eeuyCzCkQQlz3JBEkaT6MIiqOXsa1mTZNsDu459M19Jn2L10m/5MHwQkhhP9kmgiUUk8qpUrmRTD5qkor4kvWYZBxWaZND8ZcJfyIa7jpNZvD35EJIYRf+XJHUA7YqJT6SSnVXakbdBS9UljbjKSp4RCt1O78jkYIIfJMpolAaz0eqA18CQwD9iul3lJK1fR2nFLqK6XUaaWUx8WBlVIdlFIXlVJbE39eyUb8uavpYM7pIgwx/ZWlwxLsMtNYCHH98qmPQLuW6zqZ+GMHSgJzlVLvejnsG6B7JqdepbVumvgz0ZdY/MoSzJ+G9txuCKdkBusUeFJn/J9+DEoIIfzLlz6Cp5RSm4B3gX+BRlrrx4DmQL+MjtNarwTO5VageWW+6oRFObjTh05jIYS4EfhyR1AK6Ku17qa1nqO1tgForZ1Arxxev41SaptS6k+lVMOMGimlRiqlwpVS4TExMTm8pHdxpeuz01mNIcYlGPD9kc/pS5kPOxVCiILIlz6CCUBppdToxBFEN6XYl5Ne1c1ANa11E+AjYJ6XGD7XWrfQWrcICQnJwSUz99XQFiS0eYZahuP0M670+bhWby1jz0nfHycJIURB4cujoZeBGUBpoAzwtVJqfE4vrLW+pLW+kvh6IWBWSpXJ6XlzqnQRK826D2OrsybPmuZiJcHnY7ccveDHyIQQwj98eTR0P9BSaz0h8e6gNfBATi+slCqfNBRVKdUqMZazOT1vrlCKt2yDqKDO8Zhpvs+Hjf1lBytknWMhxHXGl0RwHAhK8d4KRGd2kFJqFrAWqKuUilJKPaSUelQp9Whik3uACKXUNmAqcG/i6KQC4WBwE/5yNOdB458UJdbn4zbLXYEQ4jpj8qHNRWCnUmoJoIGuwAal1FQArfVoTwdpre/zdlKt9cfAx1kLN+8EmY1Mie1LF8tmHjPN5137vT4dd2POthNC3Mh8SQS/Jv4kWeGfUAoWpWCnrs4C583cb1zCx/Y+xKa6Mcr4OCGEuJ5kmgi01jOUUhagTuKmvUlDSG9khsRP9K/t3bnTuo6exnXMcXTI9LgPl+5nxK01KGL1JccKIUT+82XUUAdgPzAN+ATYp5S6zc9x5TuLyfWn2axrc9BZgaHGv1A+ziv4Yf0Rf4YmhBC5ypfO4snA7Vrr9lrr24BuwAf+DSv/PdExqZSS4nNHL8IMkbQz7PTp2N0nLhM6ZgGr9rsmv12ITSB0zAKW7jrlp2iFECL7fEkEZq313qQ3Wut9gNl/IRUMRazJv+I8RzvO6yI+lagG+HWLa1DVpysOsmz3KfacvAzA56sO5X6gQgiRQ74kgk1KqS8Sq4V2UEpNB8L9HVh+a1qlBADt64RwW4MqzHR0podxA83Ufp/PsebgWR6aEU7aQbHT/j7AkbNXczNcIYTINl8SwaPALmB04s8u4DF/BlUQhBS1EjmpJzMebMWng2/i1+ABnNQlec38jc99BZ7EXI7nvcV7uf/L9bkXrBBC5IDXRKCUMgLbtNb/1Vr3Tfz5QGsdn0fxFQgmowFtKcJbtkE0NhxmsI+PiDxJmjN3LUHWMBBCFAxeE4HW2gHsVUpVzaN4CiwNzHe2ZYOzLm+YvyZMZf15v3L/jxBCFBy+PBoqiWtm8TKl1PykH38HVjApnrU9xjVt4S3zl5ixZ/tMZ67E88WqQ3ywZF8uxieEEFnny6ynl/0exXXA4XQ90onSZXna9jj/s3zIM6a5PpWe0CT3FqsUtwRvLHBV8R7duTZGg9wqCCHyhy93BD201v+k/AF6+DuwgqZ1jVLu14udrfjR3oHHTfO5xbAj02MHTU/uGI6Ivphuf1KSEUKI/OBLIujqYdsduR1IQfd6nzBWvdDR/X6CfRgHnRWYbP6U0qT/cM/I8G82ptu2PUoqlgoh8k+GiUAp9ZhSageuMtLbU/wcBjL/GnyDsZqMVCkV7H4fj4UnbU9Sgiu8ap7h0znWH/a8hPOJi7mzzGWC3ckDX66XxCKEyBJvdwQ/AHcC8xP/P+mnudZ6cB7EVuDt0qFMs/fhTuM6uhg2Zfs8T87akivx7Dt1mVX7zzDm54DL00KIHMgwEWitL2qtIxPXFYgCbLhGURaR4aTJRek+dfTmgLMiX1gm0ygbQ0qTvDh3e26FJoQQWeJL9dFRwClgCbAg8ecPP8dVYFVNfDw0Y3grAGyYeNH2MAC/W8djIXsVumeHH3N3Go+ft4OfNh7j+IVruRCxEEJ450tn8dNAXa11Q611o8Sfxv4OrKCa+1gbZo64mXLFrO5tm3Rd3rMNAGCs6Ydsn/uDJfu4FGfj+3VHeeHn7bSdtDzH8QohRGZ8SQTHIAvDYm5wZYsG0a5WGczG1H+6aY4+fGm/g+GmxXQzbMjWuVcdOIOWyhNCiDzmy4SyQ8AKpdQCwF1jSGv9X79FdR0om+KOIMkk+300N+zlf5YP6RH/Frt0aJbOue3YBZpM/CvHseXGcpmxCXbsTk2xoBu+4rgQAc+XO4KjuPoHLEDRFD8BzWoycmvtMqm22TAxyjYagPmW8dxpWJPj68TZHFk+5nJc9ktfJLn1nb9p/GrOk5IQouDzZc3i19JuU0rJgryQbp0BcJWg6BY/iUnmL5hs/pTajii+sPfgEkWydY3NR87TtlaZzBumcPRcbLauldLZqwk5PocQ4vrgbULZ6hSvv0uzO3sPwW8w79zjuc98r67K0IQXWO+sz2jTPLYHjeQBY/a+XQ/6Yr0scSmE8Ctvj4YKp3gdlmafVEgDKpUoxGf33+Rx3yWK8IDtJQbGu2r2vWaaQW/Dv9m6zohvfVsQztMdihBCZMZbItAZvPb0PmB1a1je6/71uj71475ik67NVMs0frZMwErWH7torTl/NYGPlu1n/rbjvPTrDs57eXzz186TPDN7K6FjFrDl6PksX08IETi8PesvoZS6G1eyKKGU6pu4XQHF/R7ZdUIpRYtqJQk/kvGH7TWCeCBhLF+a36edcSe7rMNpFz+Vk5T2+TrVxy5Mt+3YuVi+e+hmwFXBdP62aPe+kd8ll7z4YvVhpg0q6fO1hBCBxdsdwT9Ab6BX4uukWkO9gJX+D+36MaxdaKZt4rAy2DaOcbYHMSrNQutYinElR9ddtf8MF2NdM5m/Wn2Y6asO5+h8QojAlOEdgdZ6eF4Gcj3r1bgiU5ftZ9+p5A/2MkWsnLmSfmnnmY4unNIl+cIymcXWMdwa/yF2n6ZzeDb06w3UDCnCzuMZz/mTDh0hhDe+zCMQPqhSMrlE9Tv9GjHvibZ891Arj22XOpsz2XYPFdQ5vjBPxkjW5wok2XrsAj9vjmLPycvZPocQIrBJIsglH9zblCqlCgEwsGVVKpcM5hYv4/8/cvRlvG04HYzbmGD6Ful/F0LkF5kYlkuKBZlZ9UKnVNuUUlQpVYhj5zxXEf3e0YXKKoZHTX/gRPGFoydROsQv8WmteWHuduZsiiJyUk+/XEMIcX3ypQx1f6VU0cTX45VSvyilPA+eF+kseaa9l72KSfb7+M7ehWGmv1htfYq3TNMpyaVcjeFSnJ2xv+xgzqYoADZ5GeEkhAg8vjwaellrfVkpdQvQBfgS+NS/Yd04gsxGXuvd0EsLxcv2B+kYP5lfHLcwyPQ3W4IepY9htZdjsmblvhh+3HjM/X7rMddSlhev2YiIlsKyQgQ6XxJBUk9mT+BzrfUCXAXohI+Gtg3NtM1hXYFnbY9zb8J4TusSfGj5hOnmyQSTO+sZezJo+jp6fZR7CUcIcX3yJRFEK6X+BwwEFiqlrL4cp5T6Sil1WikVkcF+pZSaqpQ6oJTaLo+bXNY5G3Br/If86WhJV+MmfrJMzPF8g7ScTk1sgp2dx12PoDZGnsvV8wshri++JIIBwGKgm9b6AlAKeN6H474BunvZfwdQO/FnJDf446aPBzXjiyEtfGobj4XHbM8wPOF5aqsofrC8RVFyXlE0yZsLd9PglcXu9/0/W+vTcduOXeDERVk+U4gbjS+JoAKwQGu9XynVAeiPD9VHtdYrAW9fNe8CvtUu63CVsajgQzzXpV6NK1K3fPIyDvUrFGNy/yZej/nb2YznbY8SZohknfUJiufynUFKr/2+E+2hat2Rs1cJHbOARREnuWvav9z6zt98vvIg01ce8lssQoi85Usi+BlwKKVqAZ8DVYDsL8ybrBKuZTCTRCVuu2GlXDls9iOt6XtTJeY82sbrMfOdbRljG0FhFc8n5inUU0cJU4dQ5O6all//G8npy+lnQm+LcnUm/779OAB2p+athXt4c+HuXL2+ECL/+JIInFprO9AX+Ehr/Tyuu4Q8o5QaqZQKV0qFx8TE5OWlc1Xpwq7lLd/oE0axIDNKKVqGlsr0uB8dnXjO9gitDHtYZB3DH9bx7LEO5x7jP7maEAZNX5du24bDZ3Pt/EKIgsmXRGBTSt0HDAH+SNyWGwvZRuO6u0hSOXFbOlrrz7XWLbTWLUJC/DPhKi8UshiJnNST+1tXS7X9hxE3Z3rsXEd77k0Yzwe2frxnG4BV2Xjf/D8OB93Pq6ZvKOf1KZxvDsZc5X//HEz1iCg2IXHQmEx8FuKG5UsiGA60Ad7UWh9WSlUH0q5Ylh3zgSGJo4daAxe11idy4bzXHV+Xotyk6zLF0Y9pjj5Uj/ue1233s9NZjWGmv1gfNIpVlqfoYNiSo1je/nMPK/efSbfd5vB+52F3OLkab2dj5Dk+XLovRzEIIfKW8tRBmK6RUhagTuLbvVprmw/HzAI6AGWAU8AEEu8ktNafKaUU8DGukUWxwHCtdaZLcbVo0UKHh/u2Ytf1JHTMgnTb/h3Tic9WHOS7dUe8HttEHeAny+tYlesfy7CE51nhbJYrcfW9qRK/bPZ4o8basZ3oOXU1N1UtgcVkYOGOk+59UsZCiIJFKbVJa+1x6GKmiSBxpNAMIBJXReMqwNDEUUF57kZNBIdirhB94RoPfJk8ICvpw9RTkvDESgJ/W5/FjIOhCS+yS4fmOK5+N1Xm581RWT5OEoEQBYu3RODLo6HJwO1a6/Za69uAbsAHuRmggBohRbi1dgjfPui5dLUv4rEwMuFZnCgWWl/iQeOfOY4rO0lACHF98SURmLXWe5PeaK33kTudxcKD2+qE8O+YTqmGlTav5vsykxG6BoMSxrHPWYlXzN9xv3EJ+dHTKxPPhLh++JIINimlvlBKdUj8mQ7ceM9mCpBKJQqlGlZqyOISYwd1JfomvMY6Z33eMH/Ncst/eMY0hzLkXYG5Nm8vz7NrCSFyxpdE8CiwCxid+LMLeMyfQYnUVOJik093qe3zMVcIZlDCOCbYhlJMxfKU6VcWWMdSVx31V5gZemvhbp6clbPRTEII//GaCJRSRmCb1vq/Wuu+iT8faK3TT0EVfte6Rmn360olCmXa3omBGY5utIj/lCEJL6JR/GqZQAfDVn+Gmc7nKw/x+7bjeXpNIYTvvCYCrbUD2KuUqppH8QgP2td1TaKrWDz5w//Hka15sXs9H+TZDlAAACAASURBVM+gWOlswr0J4zlHUb40v0dvw7/4u+/gwGlZR1mI64Evj4ZKAjuVUsuUUvOTfvwdmEj2WPuarH+pM1VLB/NC97rUr1CMKqWCeaxDzSydJ1JXoFv8OxzWFZhqmUZk0GDaGHb6KWpSDYUF+GH9UUbMyHr30pV4O4fPXM2tsIQQafi0QhnQC5iIayhp0o/IIwaDolyxIAAe71CLP5+61Wv7vs0yrt13lUL0SZjISkcjAGZZ3uQOw/rcCzaFExdTL6rz0q87WLr7FAAOp6b560v4xYfhqYOnr6Pj+yvYdOQcV+LtfolViECWYSJQStVSSrXTWv+T8gfXimUyuLwAKxHsfQG5KwQzxDaWJnGfE6OLMc08ledNP2Im9z9kl+85lW7bgM/W0vfTNZy9msArv7nuSPacvORu63SmfmSVVAG136dreXzm5lyPUYhA5+2O4EPwuIr6xcR9ogCqWirY5+GmFynCbfEfMtdxG0+Y5rPJ+ggbrY8xwTSDzoZNubIYzoPfpH8UtCHyHNsS101OKs3d/cNVPPhNOKv2x1DjpYXsiPI81HXXcVljWYjc5i0RlNNa70i7MXFbqN8iElky9g5Xh/GojrUAaF8na9VZrxHEC/ZHGGt7iGO6LCHqIsNNi/nSMpkdQSPoY/DvmsaX4+x8uuKg+/2y3acBCD/iuZqqw6n57197uRSXabkrIYSPTF72lfCyL/OxiyJPDG9XHaNBMbRtKPfdXJWyRa188vdBj20faF2N3k0relyacpajM7McnSnOFeqqY/Q1ruJe0wo+tHzCh3zCx/a7+MbenTMUz/Xf4Z1Fe9yv7U5XlVNj4m1NRHTqO4DzsTamLj9AzJUE3u7bKNdjESIQebsjCFdKPZx2o1JqBLDJfyGJrLCYDIy4tQZmo4FKJQphNhp4tEMN7mxSMV3biXc1zHQhnIsUYYOuzxj7SGrFfcv/7D2xaSOjTL8RHvQYb5q+pLLy3+JA369zTXhLSgS9PvJ8RxJvc/gtBiECjbdE8DQwXCm1Qik1OfHnH+Ah4Km8CU9kh9Vk5OFbq6fbrlTWalXYMfG2fTD1479mjG0E65z1GWxaxiLLi7xh+pJCxGV+kmwyZdLRsXzvaaIvXOPc1QSv7cIjz9F04l9cvCaPkoTISIaPhrTWp4C2SqmOQFji5gVaaykicx0onzjctGkV11oBz91eN9vnsmPiR0cnfnR0oro6wSjTPAYZl2PEyVh7upvGXGFQiiNnM547cCHWRrtJy1EKDr+dccnrKcv2cyHWxpaj5+lQt6w/QhXiuuetjwAArfXfwN95EIvIRWWLBbH1la4UCzJjyGrVOi8O6wr8x/YYZ03FGGlaQG1DNKMSnuQkpYDcu86WYxcY92tEpu20hjcX7GJczwYe9yfdBS3ccUISgRAZ8GVCmbhOlQi2eEwCPz/Whs/ub87Bt3pk+9zv2wewwNGKFoZ9rAt6ksigwUQGDaKGyp2aQj+sP0pCJstjJpm+6jAfLdsPwKgfNvN9ihXdkn77n8Jl6osQGZFEEICaVytF97Dy7g7Z7EjAzBO2p+kR/xbLHM3Y53TNZl5ufY7Hjb+h8O1DPLdMXuJaJ/mP7ScYPy/5TiKL3SJCBCRJBAKArg3K+VTRNK1dOpSHbM9ze8J73Bb/ASsdjXjBPJtvzO9SJBcmpOVURnnA7nASfUEWzxECJBEEvOKFXIvNVS9TmH/HdGLELelHG/nqqC7HENsYxtke5BbDDiKCRjDfMo6hxsUY8f9wT09rO6ccKRU6ZgEvzt0OuOYutJu0nNOX/DfySYjrhSSCAPdA62oAFLG6xg1UzMZdQWqKmY4uDEx4mVn2jjhRvGaewa+WV6il8v45fdo7gtnhx9Ba888+11yIc7Heh59m5P3Fe2n79rIcRidEwSCJIMDpxDUJfO0ueKR9DZ/ahet6jLU/TJ+E13k8YTQV1VmWWl8gMmgQI4wLsJK9D+Cs8tRH8M6ivew7dcW1P0WqcDg18fb0dy4Op2byX3u5kCJpfPz3AY5flLsJcWOQRBDgyicudlO+uG93AmPvqM+SZ25jz+vdfbyCYqGzNT3i32aBoxUA480zWW99glnmN+hvXIEFGxb8M+Er6nz6foDP/kkuwWFzONHalQxH/7iFuuMXpWu/Yu9pPlp+gJd/S792w187T+ZitELkD0kEAW5wq6p8MaQF/W7KeA2DtGqXK0qQ2Zil65ymJE/YniY07gfuSxjHMudNtDHu4j3z5+wLGsq+oKGMM31PMXJnAZqdxy/S6NXF7DnpfZW0Xh+t5p1FewFYsP0EgDsxJLEnlsX2tNzmvlOX0Vqz5ej53AhbiHwhiSDAGQyKLg3KuTtVC1lcH/BPdEy/+tmsh1vnyjXXOhvyH9tjNIv7jOn2Hqx0NOKYM4SHTQtZZX2KHoZ1OR5+2nPqai7H+ba+wmf/HGTWhqPu99vTlMD29tRMKcXPm6O5+5M17kQixPUm05nFIrD0b16Z87EJPNiuOuWLF+LlFGPy29QsnenxYZWKERHtaRmL9M5TjDft97vfdzWEM8U8jU8sU1nuaMor9uEc16Vx5sH3lbG/JFdcd6a5I8hsVbSDMa7+hkgvJTGEKMjkjkCkYjIaeLxDLYLMRh5oXY3tr96e6TGd6pVl3djO7Hm9O/OfuIUu9dOXcqhYPCjT8yxxtuD2hHf53t6ZDoZtrLY+xaGg+xlp/B1DHk9QS+nZn7a5X+87lf5Rk8xZE9c7SQTCq6JWE7fUKsMXQ1pk2OaNPmGULx5EkNmIwaCYcm+zVPt/ebwta8Z29ul6UTqE8faH6JTwPp/ZewHwknkWh4LuZ7L5E0I4nysrp3kz9pcdGd4FTF95KNX7yDPJdwEJ9vTJalHECeaEH8vdAIXIZZIIhFdKKb4fcTNdGpTzuL922SLp5h6k7UhuVCnri9lE6gpMsg8iNG4mz9keIUYXp59xNRuDniDc+igfmj/mIeNCvwxD3XPyMmETFvPD+qPp9m06krpTOMHhJOZyPOCqdJp2veVHv9/M84mT2IQoqKSPQGTbP893oFRhS7rtRoPCajIQb3fyw8M3Yzbm5PuGYq6jPXMd7WlriKCdIYLWht30NKynj3ENL5u/Z7ezCpuddVjtDGOHrk6Uzp0qoy/9uiPdYy6dps1vW4973e/JpiPnKFs0iCqlgnMWoBC5RBKByLZqpQtnuG/9S53Zd+oKrap7XxEtK9Y4w1jjDHO/b6V285L5B4pxlcGmZQzGNdP3mYTH+NV5a65cs9VbqWcPpx1a6qtlu09x9koCA1pWod+nrqVCIydlvI6CEHlJEoHwixLBlgyTwKyHW3Pf9HU5vsYGXZ8+Ca8DUEMdp6VhL++Yp/OB5VM+4FMAZti7Mt/Rlk06+wvzpKRxFazLSJzNwYVrNk5ejMNsTO5GfmhGOOAq7idEQSOJQOS5BhWLpds2ulMtpi4/kO1zHtIVOeSoyO+ONow3fc8gk2shvaGmJQw1LWGmvTPj7cPROewWuxrv4OXfMl4w55HvNrH6wJkM93ua6SxEfpPOYlEgPNK+JsWCTHwy+CYO5WDBnFiCeMk+gtC4HwiNm0nn+PcAGGxaxi+WV3nUOD9Ho44S7A5mbch4FJC3JCBEQSWJQOSZMkWsrhcagszJ/+o927UOha0mtr/ajR6NKuTi0pqKg7oSoXEz+dXRjjB1mDHmH1lu/Q+3GrI3kqekh87xnJi7KYp3F+3J1XMKkVV+TQRKqe5Kqb1KqQNKqTEe9g9TSsUopbYm/ozwZzwif/04sjXPdKlD8WAze16/gz+evAWAzh4moN3bsor7daEs1jVKT/GM7QnqxX/DgPiXOaeL8p1lEv81f0IhslZBtE65ojmK5PNVqechPDdnG5+sOEhE9MUMjhDC/1R2R0FkemKljMA+oCsQBWwE7tNa70rRZhjQQms9ytfztmjRQoeHh+dytKKgSbA7qTP+TwC6NSxH35sqcyE2gRd/3pHJkZkrxlVeNP3IfcblJGDiDfv9HNQVuaatbNU1yau5wiNuqc4Xqw+73yeNIoqIvki83ck/e08zdfkBDr7VI9WyopfibBSxmHLxzkkEAqXUJq21x5mh/uwsbgUc0FofSgziR+AuYJfXo4QALKbkm1WFolvD8izOpZLPlyjMOPtDLHS24lvzJN4wf+3e9729M+PtD5IXySBlEgB4c8EuhrQJpddHqwGwJM6/OHkpzr2M6KU4G41f/YtH29dkzB31/B6jCAz+TASVgJS9alHAzR7a9VNK3Ybr7uEZrXW6njil1EhgJEDVqlX9EKooyEoVcT2XN+TySvT/OhtRJ/5buhs2YsbOTYb9DDEtoZy6wGpnGG0NO7Fj4L/2/hzUvpfpzq7pqw6zITJ55nJC4jDVdpOWu+8WLlx1rdvwx/bjkghErsnvzuLfgVCtdWNgCTDDUyOt9eda6xZa6xYhISF5GqDIf+N71gegZWhJjzOZ0/K0aM7TXWp7bOvAyAJna+Y5b2GCfShT7HfTxrCT18wzuM2wnS6GLfxleYHnTLOprGJy9ov4wOahXhHAB0v2pSpfEWdzeCyAJ0R2+POOIBqokuJ95cRtblrrsynefgG868d4xHUq2OL617REsIXNL3dNt0h92aJW3u/fhCZVSiSOSErfufx0lzp8uHS/1+toDHxg78+n9t60MuxhpzOU4uoqU8wfM8r0G6NMvxGtS3NCl+aqDmKCfSiRukLu/aKkL4GdZMqy/dQqW4TGlV11m85cSeD2D1bSs1EFptzbFFOOyniIQOfPf3s2ArWVUtWVUhbgXmB+ygZKqZT/FfUGdvsxHnGD+Hp4S/o3r+x+/+bdjbitTgjFC5kpHmzO8fnjsLLS2YSzFOeQrkjvhDfoFz+Bt2z3sctZDQNO2hu3M9/yMj0N6+hvXEEIF3J8Xcg4EQAcPnOVtLsX7DjBWwtl+KnIGb8lAq21HRgFLMb1Af+T1nqnUmqiUqp3YrPRSqmdSqltwGhgmL/iETeOjnXL8l7/Jl7bjO5cG7NRMeKW6rSoVhKARU9nr/6QxsAmXZfPHXfysO05+iZM5J74VzDgZJplKu+ZP2ed9QmeNP6SrfOntO/UlQz3/XfJPo/bj1/IeLay06lZtT/GXSNp4Y4T1HxpYbZrJokbk19LTGitFwIL02x7JcXrscBYf8Ygrl/dG5ZnkZeRQp3rlWXZntMex/c827UOz3atk2pbvfLFeKJjTab9fdDDEVkTrutxR8Lb1FdHidEleMY0l/+Y5xKiLrLI2ZIIZyiXKJLj66Tl8PABfupyxnMhek9bTUT0JT4ZfBM9GlXg8ZmbAYi+cI3KJaX6qXCRWkOiwPpoUDNi4x0Z7m9XqwzL9pymWmnfP9Ce71aPOuWK8tSPW3Mc3zFdjmPaVURuuO0F3tJfMsS0hCEswakVfzpbMtfRnr+dTcmt4aidJ/+TbtuWoxe49/O1fDm0JQ0nLMZiNLDvzTuwOZzuZUOjzvt3MR9xfZMeJlFgmY0Gr8/8h7cLZf1Lnamdxdm+dzWtlOUS0G/3beR1vwMjL9pH0in+fcbYRvC1ozttDLv42vIesy2v01Rlv6CeL9YdOsf5WNciPQkeqqOm3aR16rWYE+xOnx8XdXx/Bfd/sT77wYoCRxKBuG4ppShXLPO1kDPTPLEPISNzHm3Dfa18m79ySFfkR0cnXrc/QMv4T3nJ9hA11AnmWV/hH8vT9DasoTD+qUD686bkQXnfrTvClbjkD3qH08m0v5OTUef//kPYhMXEJthxODV1xv/JxD98m+t5+MxVVh8443EFN3F9kkQgAlZIUVcRvFoh3p/ltwzN3uI6Doz84OhM+/gP+J+9J9UMp5lq+ZjN1kd4xjSH8pzN/CRZ8MHS5M7kl+dF0Oz1Je73dqfmvcV73e+T1le+Gu/gg8RO6O/XHXHvn/j7LkLHLPB6l/DSrzkv9yEKBkkEImAtHH0r855o504IA1tUyeSI1JpVLeFTu1iCeNs+mMZx0xmR8B9O6ZI8ZfqV1danmGb+kEeMvzPEuJgKuZwYUnJm8Hne7cOVfJx4p2BzJDf66l9X+YsV+7I3iS7e7uCr1YdxZHRhP9I69UgpkTlJBCJghRS10rRKCdrXdc1WH9Cycro2g27O+JFQVj9nLlGYpc7mdE14j0EJL7Fd1+BWQwRjzbOYaJ7B2qAnWW0dzSbrI2y2jmS2ZSLzLeNoa8h4IRxfHT171eP2c1cTUr0/cvZqqgl7w7/emGr/tmMZz5c4eyWe7xLvKqYtP8DEP3bx0IyNGbb3l3lbo3ngyw38uDHjdSNEapIIRMBrGVqKyEk9aV4t+RHQpL6N6FSvLK/1bpjhcSnzgK99CADxWFjjDKNvwkQax0+nWdxnvGh7mL3Oymx31iBal+GqLoRG0dhwmB8sbzHDPIkm6kCaq/ruUor+Am++WRPpdf9d0/5N9b7T+yvcr5+evZWX50Xw3bojbElMGCv2pr6jWLH3NKFjFrjnPny37ggbDp/zKTZfRSeuAicjpXwnw0eF8ODeVlW5N82He6nCllTfoIe3DeXp2a5hqI+2r8GsDdnpPFWcpxizHR2Z7eiYbm8xrnKfcTnPmObym/UVonQZLutCTLH3Y5Gzlc9XWb7ntE/tvv43Mt22BLszVTXYlA6dSb7TSPrbvDwv4zuYHxNXd9t67AIVSxRyt83qKC5v5IlQ1kkiECKNpD4Db+Y+2oYWoaXo0yx9VdIyRSw0rVKCpbtP88eTt3DqUpx78fqsukRh/ue4k58dt9HNuJF2hggaGw7xmeVDljma8an9TsK1f6uQ1hn/J2WLWlnyTHuP+7XWKB8rw+ZyAVnv18qjdSVuBJIIhEhh2X/aUyrYc4VTU4qFYJIK4aX09bCW/LY1mg/vbZZqe1il4jmO6wzFmenowkxHF8zYecT4O8NNi5hrncg2Zw3etw9glbNxjq+TkdOX42ky8S+P+x6aEc5Xw1pmeOzVeDuFra6/V9JM8cNn0vdZrNwXQ1il4j5VmBW5S/oIhEihZkiRDNclnjkieTmNcsXS3zV0rFc2XRLIzE+PtMlagIANEx877uaW+Cm8axtAGXWR7yyTWGgZy9fmd3jJNJP3TJ8x0Pg3oeoEBjyXts4ty/ecxunU7Dx+yeP+hhMWA67S2UmPbVIOZU3aN+SrDQz9agMxl+MJHbOAf9KMWIo6H8tKH0YxZffJ0E8bjzF1mfcKtTcquSMQwke1yxVl7xvdOX/VRukimT8+8oXZqJj1cGvum74uy8deI4hPHH2Y4ejGvcbl9DKu52bDHjqqbTi1or9a6W57wFmRk7okK52N+d7RlVhyPhEvpbWHvA99vZbgYOR3qR+PxSYkd2C/scA1me3A6StEHHet3/zV6sO0r5O8/kjnyf8Qb3f63J+Q1cdQL/y8HXAVLAw0kgiEyAKryUj54unXO8jM18NaMvybjdSvUIzdJ5K/OZcItlC9TOEsn++R9jX43z+HALhKIb509ORLR/IHpMJJLXWc5oZ9dDZsoYS6TDl1gZfMs3jBNJutuhZLHM1Z7Qxjl66GzuHDgTnh3odq1n9lUbptDV5Z7H79/TpXR/s1m8M9ZNWWpi5GfAaL9qSVUWfxiYvX2HzkAj0b5+4aEjcCSQRC5IGO9cq6v8nG2RwoBbuOX8pWEgDoXK+cOxF4ojGwX1dmv6MyPzo6ubc3U/vpbtzgnr8AsNrRkPH2B3O0yM68rcezfWxGLl6zsffkZb5ZE0nnemXd20PHLKBsUSsLRt9KnM1BlVKuooMz1x+hfoVi7gqtCteIpxMXr1GtdGHavL0cgG4N70i3kE9E9MVcjz83LIo4wY8bj/HNcN9HiGWHJAIh8ljSCmrNqibXOOpSvxxLd5/y2H7MHfWY9GfqxWdKF8leh+oWXZst9tq8DVRRp+htWMto06+ssP6HSGc59utK7NLViNEl2OkM5QzFOKbLklvVU7PiQqyNh2ZsJOr8tXRDc09fjqflm0uB5KGn435NPWz15KU4xv6yg583R7Ftwu3u7btPXKZR5dQd+L0+Wu1+/fee03RMkXjy06Pfb86T60hnsRAFQEhR1wf7LbXKuLc1q1qCtWM78Wj7munaW4wGejaqwDv9vFdF9eaYLsc0Rx9ujf+QN22DOKZDCFWneMr0K2+Yv+ZX6wRWWZ9ho/UxvjW/zUPGBVRTJ8l+d2zWRF+4xtkrCZm2i02wY/dQcfVgzFV+3hwFwP4U6zvf+fFq/k4xr+LBb1LPfv5o+X4OxbgWCIq5HE+czVUKfd6WaIZ8tYHzVz3HZHM4+Sn8WKq1pX0ReeYq3ySW9MgvckcgRAFQyOz6T7FplRKsPnCGaqWD+fXxdhm2r1IqmGmDbwLgxZ9zVvztNCWZ7ujFdEcvAEpxiXqGo9RUx3FioL1hG/XUUW4z7+BlZrLDGcprtiF+n78Arj6DzKTsa0hp05Hz7tf3fLY21b6DMVfc3/rTTrbbfPQCnVKs+1C7bBGKFzITnni+79cd4UkPHcqfrjjIf5fsw2xU3N0suVyJ1prXft/Fva2qUK98sXTH3fPZGs5cSeDeVlU9rredFyQRCFEA/Of2OhQvZOaJjjUxGRX9bkpf9wjg9gbl0m2rV74oe066vvHWKVeEGmWKeF3ZLTPnKMYaZxhrCANgpqMLoGmh9tLCsI9BxmXMtU5kr7MyJ3UpLlKYHc7qnNKluEwhbJiork6wy1mNTbputuPwJ6fWjPl5u09lzPefTr18qKcv/LEJdvYl3nVcjLWl2nfqUjzfrInkmzWRrHqhI1azgStxdmokVr09n9jeaMj48dug6etoGVqKp7vU9nnyXlZIIhCiAChsNfFUF9e3zKe71Em3v0SwmQuxNj4f0iLdvsc61HSvuPbwrTW4q2klLsfZGPC/tRyMST9xK3JST+7+5F+2HM24gFx6inBdj3BHPX5wdOJF02xqGaKpaThOEAn0Nq5Nd4RTKz539GKRwzXZzIQdOyb26CrEkTvDb7NLa7JdlO7MlXhOXLzGyn0xDGxZld0nLnHHlFXu/R8s3c+wdtXd71OOfhr53Sb3qLEN4zqz7dhFd4VWg5cP+DUHz7Lm4FlKFbYwtG1otuL2RhKBENeBf57rSNQFz0XU7mpaic1HzjNj7RGsZiMWk4HSRayULmxNlwiWPusqE9GnaaUsJoJklyjCOPtDqbaV5BKl1GWqqNMUJp6tzpo8ZfqFR02/86jp91RtHVoRrcuwVddipr0Lm3VtbHn8UbQ4B3dM36074q6yGh55njJpSpJcvJb6juD2D5Lnc6QcOtzqzWWp2i3YcYLeTSp6vfaiiJOSCIQIVMWDzRQPzrhUxQvd61G2WBA9G3kfAlqrrOtxxJA21QirVIxv1hzh922ph372bFSBBTtOZCm+8xTjvC7GQZ1ce+kF+yN87OhDE3WQKxTCjpEiXKOu4RgN1RE6GLbR27qWa9rCRmddNjrrstTZnIO6IglkvERpbticzSSY1pxNUZm28aWfA2D0rC3uROB0andHd0pOP1XUk0QgxA2gsNXEEx1rpd6Y+KRhWNtQvlkTyQcDmyTvUorm1UrRvFqpdIng40HNePVKQxIcTtpNWp7+WhYjVxN8+3A7qstxVKfu1/jT6SrVUZhr3GLYQQfDNtoYdnGbeQf/YS7x2sQhXZFDujwV1Dm2OmuxX1dij7Mqdoxo4LCuwFUK+RRDfthy9DzNqpbkt63RmTf24OnZW5m/Lf3cDH+N15JEIMQNqlbZImw4fI6BLavwqpd1FSxGQ6oF75VS7gqsh9/uQfWxC1O13zmxe6rFa7LrKoVY7GzF4sRy2lXVKVoZ9lBLHae+OkJrw25O6NIMNi7FqtKvpxCtSwNgxsFlXYgrFKKCOschXYHv7F35y9ki00dOHQxbaGY4QISzOsudzXCQO6N27v5kTbaOy+zvGu/j3UVWSSIQ4gb1Sq8G3BFWnvoV0g9ZTGnP690BeG7uNlbvP5NqX8oRKmWKWDiTOK5/5fMdmbPpGB8tP5Br8R7V5TjqSD8qyoSdOiqKsuoCFmwU5RpVDaeoomIw4MSmTRRTsRQllm26BvXUMaZZphKvTSx3NuMbe3dqG6JoqCIppS5zmWBKcJm6KooqhuQidjG6GIscrYjQ1fnL0ZzzeP+75YezGcxhyCl1va3r2aJFCx0enr3a7kKIrEv6lrrztW7uctLgGh+f9m6hIFA46WzYQjtDBAONKwhW8QBc1oU4rUtgVTYc2sAOXYN1zvrMcbTnNsN2+hlXcbshHINyfSbudlZhlbMxcxztOaLLEUQ8FhycpWiOazNlV0hRKxvHdcnWsUqpTVrr9MPOkDsCIYQPSgSbUyUBwOt49rBKxYiIvsSUe5u6h7bmFY2Bpc7mLHU2Z6r9bpoaDnJcl2afrpzhB/gSZwuWOFtQhovUMRyjmTpAa8MuhhkXMdKU+nHNCV2Kfc7KnNIlUWguE0xxdYVoXYb9zsoc0hWJxYoFG8HEU1xdJURdYK2zAdG6TI6SiDwaEkLki3+e70CxIN9G8dzVtCJFrCYux9mJiL6UapJU+PguFAsyYzIoaryU/k6ia4NyLNl1itfvasjLv+3MldjPU4y/nb6vEXGG4pxxFmcNYUxz9CGEC3Q2bqaqOk28NhOLlaaGA1RTp2lgiMSGieJcJZYgSnIZk8l7hdSr2kq0LsNJXYqDuiKHdAXicJUXsWJjq7MmEbpGhsfbs1i+wleSCIQQXlUrnXGF1Kc612bKsv0YDQqHUzMlcWGeM1fiKRFsplvD8u62ZVKs4fDzY23YdOQ8IUWtvLdoL8cvxlEk8Y6jVGErXw9v6S5HnVbkpJ650lntixhKpKreCkAGX8qtJFBTHae6OkkRdY0ruhBXsZKAGYWmsjpDXXWM6uoE5dV5Whr2UkilfuY/xX43EfaME0Ha0ty5RRKBECLbnulah2e61uFirI3L8ckTqcoUIKUjRwAACrRJREFUsTLxrjD3+2BL6tE4SUNXAdrWLMOuE5doXKk4JYMt3N6wnHu27aCbq/LD+uTKo095WTSmaqlgjp7zPOnOF9XLFPa4hKav4rGwS4eyS4f61N6MnUrK1VltxEk8Fo4njoTKiM3hnzsC6SwWQvjV+asJmIyKoj4+XkrrYMwVTl2Ko0nlEu5+ipovLcTh1O4FfwCaVC7OtqiLPHxrdXo3qcSdHyeXlu7fvDJzNkUxtE01Zqw9ku4ah97qgcGgqPXSQuxOTd+bKvHL5uzNAfA3X1doS8tbZ7GUoRZC+FXJwpZsJwFwrSPdtmaZVJ3V2ybczo5Xb6djvbJM6usqxf3m3Y24r1VVRnWqTaPKxYmc1JNxPerToW4I7/VvQuSknh6rhgIYEvsy5j3Rjqe71Oa/A5pSqUTyhLW2Nb1/U5/xoH8XjknyYnf/VHyVOwIhxHUvwe7EYsra99qU/QyevmVfjLVxPjaBVQfOcM9Nlfl0xQGmepg30bhyceaPuoV9py5TNMjkXgktrXuaV2auDyUpvDnwZvrV1Xzl7Y5AEoEQIiDN2xLN07O38vuoW9KtWObN1Xg7JqPCavI8C7nn1FXsPO4qLvdEx5rsP3WFjwY1w2I0oJRyJ6BbapWheLCZBduT6zo1qVKCbccuEGwxEptYxuOvZ25zF67L7mMhkHkEQgiRTp9mlejTrFLmDdNIO58irV8fb8epS3HM2xLNqE61MpxvMW3QTSQ4nNgdTsb3bECJYHOqR2hJCaNOuaJ8Pbwl1ize8WSFX+8IlFLdgSmAEfhCaz0pzX4r8C3QHDgLDNRaR3o7p9wRCCGuZ+sPnaVq6WAqFPdeNG/lvhiOnovl/tbVcuW6+XJHoJQyAtOArkAUsFEpNV9rvStFs4eA81rrWkqpe4F3gIH+ikkIIfLbzTW8dzwnua1OiJ8jSebPUUOtgANa60Na6wTgR+CuNG3uAmYkvp4LdFb+WIdNCCFEhvyZCCoBKdeCi0rc5rGN1toOXATSpUul1EilVLhSKjwmJibtbiGEEDlwXcwj0Fp/rrVuobVuERKSd7dLQggRCPyZCKKBKineV07c5rGNUsoEFMfVaSyEECKP+DMRbARqK6WqK6UswL3A/DRt5gNDE1/fAyzX19vEBiGEuM75bdSQ1tqulBoFLMY1fPQrrfVOpdREIFxrPR/4EvhOKXUAOIcrWQghhMhDfp1QprVeCCxMs+2VFK/jgP7+jEEIIf7f3tnG2FVVYfh500LBQmhLCWmCYdoEIcUonYhSraRRUy0hxBgTWkxsRKPiJxpDppKY+A81MWpiBGL8+FErCvKRJloQQbGGllI6ZUodOkoTq/TDKOBHYkhZ/tjr2tPbGTrT3rmz63mf5OSsvc65e7/37nvvOmefc9Y2r85pcbHYGGPM9HHa5RqSdBg4Po/s5FgI/PWEe80stWusXR/Ur7F2fVC/xtr1QX0aL46IcW+7PO0CwakgaftEj1jXQu0aa9cH9WusXR/Ur7F2fXB6aOzgoSFjjGk5DgTGGNNy2hYI7pxpAZOgdo2164P6NdauD+rXWLs+OD00Ai27RmCMMeZ42nZGYIwxpgsHAmOMaTmtCQSS3iNpVNKYpKE+tvs9SYckjTR8CyQ9JGlvruenX5K+lRp3SRpsvGZd7r9X0rrx2joFja+V9IikZyTtlvTZmnRKOkvSNknDqe/L6V8saWvquCtzWiFpTpbHcvtAo6716R+V9O5e6GvUPUvSU5I2Vapvn6SnJe2UtD19VfRx1jtP0t2Sfi9pj6Tllem7ND+7zvKSpJtr0njSRMT//ULJdfQHYAlwJjAMLO1T21cDg8BIw/dVYCjtIeAraV8D/BwQcBWwNf0LgD/men7a83uocREwmPa5wLPA0lp0ZjvnpH0GsDXb/QmwJv23Azel/Qng9rTXAHelvTT7fg6wOL8Ts3r4OX4e+BGwKcu16dsHLOzyVdHHWfcPgY+kfSYwryZ9XVpnAQeAi2vVOKX3M5ON9+1NwnJgc6O8Hljfx/YHODYQjAKL0l4EjKZ9B7C2ez9gLXBHw3/MftOg937KFKPV6QReA+wA3kJ5anN2dx9TEh0uT3t27qfufm/u1wNdFwEPA+8ANmV71ejL+vZxfCCooo8pKeifI29gqU3fOHpXAVtq1jiVpS1DQ5OZLa2fXBgRz6d9ALgw7Yl09k1/DlMsoxx1V6Mzh112AoeAhyhHyy9Emdmuu62JZr6bzs/xG8AtwCtZPr8yfQABPCjpSUkfTV8tfbwYOAx8P4fXvitpbkX6ulkDbEy7Vo2Tpi2BoFqiHBJUcQ+vpHOAe4CbI+Kl5raZ1hkRRyLiCsqR95uBy2ZKSzeSrgUORcSTM63lBKyIiEFgNfBJSVc3N85wH8+mDKF+JyKWAf+iDLP8j5n+DnbIaz3XAT/t3laLxqnSlkAwmdnS+slBSYsAcn0o/RPpnHb9ks6gBIENEfGzWnVGxAvAI5ShlnkqM9t1tzXRzHfTpe9twHWS9gE/pgwPfbMifQBExJ9zfQi4lxJQa+nj/cD+iNia5bspgaEWfU1WAzsi4mCWa9Q4JdoSCCYzW1o/ac7Mto4yJt/xfzDvNrgKeDFPOTcDqyTNzzsSVqWvJ0gSZZKgPRHx9dp0SrpA0ry0z6Zcv9hDCQjvn0DfeDPfPQCsybt2FgOXANtOVV9ErI+IiyJigPLd+lVEfKAWfQCS5ko6t2NT+maESvo4Ig4Af5J0abreCTxTi74u1nJ0WKijpTaNU2MmL1D0c6FcwX+WMrZ8ax/b3Qg8D7xMOer5MGU8+GFgL/BLYEHuK+DbqfFp4E2Nem4ExnL5UI81rqCczu4CduZyTS06gTcAT6W+EeBL6V9C+aMco5ymz0n/WVkey+1LGnXdmrpHgdXT0N8rOXrXUDX6UstwLrs7v4Fa+jjrvQLYnv18H+WOmmr0Zd1zKWdv5zV8VWk8mcUpJowxpuW0ZWjIGGPMBDgQGGNMy3EgMMaYluNAYIwxLceBwBhjWo4DgWktkv6Z6wFJN/S47i92lX/Xy/qN6SUOBMaUpIBTCgSNJ4Yn4phAEBFvnaImY/qGA4ExcBvw9swx/7lMcPc1SU9kHvmPAUhaKekxSQ9QnnpF0n2ZxG13J5GbpNuAs7O+DenrnH0o6x5RmRvg+kbdj+poPv4N+cS3MdPOiY5qjGkDQ8AXIuJagPxDfzEirpQ0B9gi6cHcdxB4fUQ8l+UbI+JvmfriCUn3RMSQpE9FSZLXzfsoT9C+EViYr/lNblsGXA78BdhCyWH0296/XWOOxWcExhzPKkqOmJ2UdNznU/L+AGxrBAGAz0gaBh6nJBK7hFdnBbAxSjbVg8CvgSsbde+PiFcoaT4GevJujDkBPiMw5ngEfDoijkkEJmklJT1ys/wuyuQx/5b0KCWP0Mnyn4Z9BP8+TZ/wGYEx8A/KFJ0dNgM3ZWpuJL0uM3Z2cx7w9wwCl1GmI+zwcuf1XTwGXJ/XIS6gTGXakwyjxpwsPuIwpmS7PJJDPD+gzCUwAOzIC7aHgfeO87pfAB+XtIeSLfTxxrY7gV2SdkRJSd3hXspcCsOUjK+3RMSBDCTGzAjOPmqMMS3HQ0PGGNNyHAiMMablOBAYY0zLcSAwxpiW40BgjDEtx4HAGGNajgOBMca0nP8C1ijxWD8cmGgAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SdPdWeJZd4ec",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "outputId": "1512f148-4486-42e1-c1cd-e4c73fd31154"
      },
      "source": [
        "plt.plot(np.arange(1, NUM_EPOCHS+1), train_acc_list, label='Training')\n",
        "plt.plot(np.arange(1, NUM_EPOCHS+1), valid_acc_list, label='Validation')\n",
        "\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEGCAYAAABiq/5QAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXwUVbbA8d/JvkH2hECAhH1fAygoiuC+IIgKo09QB9BRx+Xp6PPNjMyM+tRhxm1cxmVQFAUFBdyVVRAFEgj7HhIIhGyE7Hvf90c1ECFAJ6TTSfp8P598uqtSXX26Aqdu37p1rhhjUEop5T48XB2AUkqpxqWJXyml3IwmfqWUcjOa+JVSys1o4ldKKTfj5eoAHBEREWHi4uJcHYZSSjUrSUlJOcaYyFPXN4vEHxcXR2JioqvDUEqpZkVE0mpbr109SinlZjTxK6WUm9HEr5RSbqZZ9PHXprKykvT0dMrKylwdSovg5+dHbGws3t7erg5FKeVkzTbxp6en06pVK+Li4hARV4fTrBljyM3NJT09nfj4eFeHo5Rysmbb1VNWVkZ4eLgm/QYgIoSHh+u3J6XcRLNN/IAm/Qakx1Ip99Fsu3qUUqq5KS6vIiO/lMPHysjILyWroJzW/t5Et/YlqrUfbVr7EdnKF29P57bJNfHXU25uLqNHjwbgyJEjeHp6Ehlp3SC3bt06fHx8zvjaxMREZs+ezSuvvHLW9xg+fDhr1qxpuKCVUo3CGMPGg8f4ZksGe7OKyMgv4/CxUgrKqhx6fUSQD1Gt/Ihu7ctfx/ahfVhAg8anib+ewsPDSU5OBmDGjBkEBQXx6KOPnvh9VVUVXl61H96EhAQSEhLO+R6a9JU6P9U2w9qUXNoE+xEfEejULk1jDDuPFLJ402G+2HSY9LxSfDw96BodRGxoAEPjw4gJ9qdtiB9tQ/yJCbZa94VlVWQWlJFVUM6RgjIyC8rILCgnq6CMzMIyp7T+NfE3oClTpuDn58fGjRsZMWIEEydO5MEHH6SsrAx/f39mzZpF9+7dWbFiBTNnzuTLL79kxowZHDhwgJSUFA4cOMBDDz3E73//ewCCgoIoKipixYoVzJgxg4iICLZu3crgwYP58MMPERG+/vprHnnkEQIDAxkxYgQpKSl8+eWXLj4SSrneuv1HeWrxNnZkFAAQGuDNwA6hDO4YysAOIfSPDSHQt/YUaIyhuKKavOIKyiqr8fXyxNfbA18vD/y8PfHx9MDDwzqJ7M8p5otNh1m86TB7s4rw9BBGdIngoTHduKJ3NK39zj5E2jfIk4ggX3q3bdjPfzYtIvH/5YttbD9c0KD77NW2NU9d37vOr0tPT2fNmjV4enpSUFDAqlWr8PLyYsmSJTz55JMsWLDgtNfs3LmT5cuXU1hYSPfu3bn33ntPG0+/ceNGtm3bRtu2bRkxYgQ//fQTCQkJTJ8+nR9//JH4+HgmTZpU78+rlKsYY7AZ8PRomNb4kfwy/u+bHSxKPkxMsB8zb+5PVbWNpLQ8NhzIY9nOLMB6vx5tWtGjTWtKKqo4WlzBsZJK8kqsx4pq21nfx8fTAx8vD4rKre6boXFh/O3GPlzTpw3hQb4N8lmcpUUk/qbk5ptvxtPTE4D8/HwmT57Mnj17EBEqKytrfc21116Lr68vvr6+REVFkZmZSWxs7K+2GTp06Il1AwYMIDU1laCgIDp16nRi7P2kSZN46623nPjplDp/1TbDjowCElOPsj4tj/X7j5JbXEFsqD8dwgKICw+kY3gAHe2PHcIC8PP2POd+y6uq+c/qVF5dtoeqasP9o7rwu1GdCfCx0tzEoR0AOFZSwcYDx9hwwDoRrN6bTSs/b8ICfOgYHsCA9iGEBHoTGuBDWIAPfj6eVFTZKK+qprzSRnmVjbLK6hOPsaH+XNsvhphgf6cet4bUIhJ/fVrmzhIYGHji+Z/+9CdGjRrF559/TmpqKpdeemmtr/H1Pdk68PT0pKrq9AtAjmyjVFNksxnWpx5l3X4r0W9IyzvRSm4b7MeFncOJCfbnYF4JB3JLWHjwEIWnXARtG+xH56ggOkcG0Tky0HqMCiKqlS8iwvJdWfz1i+3szylmTM8o/nRdLzqGB9YWDiEBPozqEcWoHlFO/+xNVYtI/E1Vfn4+7dq1A+C9995r8P13796dlJQUUlNTiYuLY968eQ3+Hkqdj8yCMh6el8yafbkAdI9uxdgBbRkaH0ZCXBjtQk5vJRtjOFZSSdrREtJyi0nLLWF/TjH7sov4NPEgxRXVJ7YN8vUiurUv+7KLiY8IZNadQxjV3X0TuqM08TvRH/7wByZPnszTTz/Ntdde2+D79/f35/XXX+eqq64iMDCQIUOGNPh7KFVfS3dk8uinmyirtPG3sb25vn9bQgLOPMz5OBEhNNCH0EAfBrQP+dXvjDFkFpSzL7vI+skqIjW3hJsT2nPniDh8vc7dJaRAjDGujuGcEhISzKkTsezYsYOePXu6KKKmo6ioiKCgIIwx3HfffXTt2pWHH364XvvSY6oaQnlVNc99s5NZP6XSM6Y1r04aQJeoVq4Oyy2JSJIx5rSx49rib+befvtt3n//fSoqKhg4cCDTp093dUjKje3LLuKBjzayPaOAKcPjeOLqHg5dmFWNSxN/M/fwww/Xu4Wv1NnYbIath/NZtjOLn/bmEODjRbfoILpGt6JbdCu6RgWdGAdvjOHTpHSeWrQNP28P3rkjgTG9ol38CdSZaOJXSp1QWFbJ6j05LNuZxYrd2WQXliMC/doFU1xezs8puVRUnRzf3i7En27RQRhgxa5sLugUxku3DqRNsJ/rPoQ6J038Sim+3pLBh7+ksT71KJXVhtZ+XozsFsllPaK4pFvkiRuSqm2GA0dL2J1ZyJ7MQnZlFrEns5AjBWU8ekU37r20S4PdiKWcRxO/Ui2EMaZetWg++DmVPy3aRnxEIHddFM9l3aMY3DEUr1pqxHh6CPERgcRHBHJl7zYNELVyBU38SjVzxhheXrqH2T+n8ey4vlzVx/GEPPvnVP68aBtjekbx2m2DdDikm2jWE7G40qhRo/juu+9+te6ll17i3nvvrXX7Sy+9lONDUq+55hqOHTt22jYzZsxg5syZZ33fhQsXsn379hPLf/7zn1myZEldw1cthDGGmd/v4qUlewC458Mk/vn9Lmy2cw/TPpn0o3n9tsGa9N2IJv56mjRpEnPnzv3Vurlz5zpUKO3rr78mJCTknNvV5tTE/9e//pUxY8bUa1+qeTPG8Py3u3ht+T4mDW3Pmicu45aEWF5ZtpepsxMpKKu9NhTA+2uspH95r2hev20QPl6aCtyJ/rXracKECXz11VdUVFQAkJqayuHDh/n4449JSEigd+/ePPXUU7W+Ni4ujpycHACeeeYZunXrxkUXXcSuXbtObPP2228zZMgQ+vfvz0033URJSQlr1qxh8eLFPPbYYwwYMIB9+/YxZcoU5s+fD8DSpUsZOHAgffv25a677qK8vPzE+z311FMMGjSIvn37snPnTmceGtUIjDE8+/UO3ly5j9uGdeCZG/vi5+3J8zf1469je7NydzY3vvYTe7OKTnvtez/t56nF27iiVzSv/UaTvjtyah+/iDwITAUEeNsY85KIhAHzgDggFbjFGJN3Xm/0zRNwZMv5BXuqNn3h6ufO+OuwsDCGDh3KN998w9ixY5k7dy633HILTz75JGFhYVRXVzN69Gg2b95Mv379at1HUlISc+fOJTk5maqqKgYNGsTgwYMBGD9+PFOnTgXgj3/8I++++y4PPPAAN9xwA9dddx0TJkz41b7KysqYMmUKS5cupVu3btxxxx288cYbPPTQQwBERESwYcMGXn/9dWbOnMk777zTEEdJuYAxhr9+uZ1ZP6Uy+cKOzLih94mLuiLCHRfG0T26Ffd9tIEbX/uJF28dwOX2MfWzftrPX77YzpW9o3l1kiZ9d+W0v7qI9MFK+kOB/sB1ItIFeAJYaozpCiy1LzdLNbt7jnfzfPLJJwwaNIiBAweybdu2X3XLnGrVqlWMGzeOgIAAWrduzQ033HDid1u3buXiiy+mb9++zJkzh23btp01ll27dhEfH0+3bt0AmDx5Mj/++OOJ348fPx6AwYMHk5qaWt+PrFzMGMNTi7cx66dU7hwR96ukX9OwTuEsvv8iOkUGMnV2Ii/+sJt3V59M+v/Slr5bc2aLvyew1hhTAiAiK4HxwFjgUvs27wMrgMfP653O0jJ3prFjx/Lwww+zYcMGSkpKCAsLY+bMmaxfv57Q0FCmTJlCWVlZvfY9ZcoUFi5cSP/+/XnvvfdYsWLFecV6vKyzlnRuvmw2w58WbWXO2gNMvTieJ6/pedbhm21D/Plk+oX87+dbeXmpdfH3qt5tePU3A50+mbdq2pz5198KXCwi4SISAFwDtAeijTEZ9m2OALXe1y0i00QkUUQSs7OznRhm/QUFBTFq1CjuuusuJk2aREFBAYGBgQQHB5OZmck333xz1tePHDmShQsXUlpaSmFhIV988cWJ3xUWFhITE0NlZSVz5sw5sb5Vq1YUFhaetq/u3buTmprK3r17Afjggw+45JJLGuiTKlcrLq/iyc+3MGftAe65pPM5k/5xft6ezLy5H8+M68PdF8Vr0leAE1v8xpgdIvI88D1QDCQD1adsY0Sk1nFnxpi3gLfAqs7prDjP16RJkxg3bhxz586lR48eDBw4kB49etC+fXtGjBhx1tcOGjSIW2+9lf79+xMVFfWrssp/+9vfGDZsGJGRkQwbNuxEsp84cSJTp07llVdeOXFRF8DPz49Zs2Zx8803U1VVxZAhQ7jnnnuc86FVnVVW23ht+V5yiyoY3TOKCzuHn3P4pDGGDQfymLf+IF9uzqCkopr7R3Xhv6/oVqcbtUSE24Z1PN+PoFqQRivLLCLPAunAg8ClxpgMEYkBVhhjup/ttVqWuXHoMXWO/NJK7puzgdV7c/Dz9qCs0kaQrxeXdItkTK8oRnWP+lWd+pyicj7bkM4nienszSoiwMeT6/rFcOuQDgzuGOrCT6KaG5eUZRaRKGNMloh0wOrfvwCIByYDz9kfFzkzBqVc6UBuCXe9v5603GJemNCPG/q35ed9uXy/PZMlOzL5aksGnh7CkLhQRnaLZNPBYyzdkUWVzTC4YyjP39SXa/u1JchXb7JXDcfZ/5oWiEg4UAncZ4w5JiLPAZ+IyN1AGnCLk2NQyiWS0o4ydXYS1TbD7LuGcWHncIAT870+Y+vD5kP5/LD9CEu2Z/HCt7uICPLhroviuSUhVicvUU7j1MRvjLm4lnW5wOgG2n+9ilKp0zWHmdiak0XJh3hs/mbaBvvxnylD6BQZdNo2Hh7CgPYhDGgfwmNX9iCrsIzQAB+9+Kqcrtl+f/Tz8yM3N5fw8HBN/ufJGENubi5+flpD/XwdL5j20pI9DI0P49+3DyY08NzzzAJEtdLjrxpHs038sbGxpKen01SHejY3fn5+xMbGujqMZq28qprH529mYfJhbhoUy7Pj+2jhM9UkNdvE7+3tTXx8vKvDUAqA7MJy7vkwiaS0PB67sju/u7SzfhNVTVazTfxKNRVbD+UzbXYiR0sqeO03g7i2X4yrQ1LqrDTxK3UevtmSwSOfbCIkwJv59wynT7tgV4ek1Dlp4leqHowxvLJ0Ly8u2c3ADiH8+78G68VZ1Wxo4leqjkorqnn00018tSWD8YPa8ew4qxa+Us2FJn6l6uDwsVKmzk5ke0YBT17Tg6kXd9KLuKrZ0cSvlN3XWzL45w+7qaq24evlia+3Bz6eHvh6e1jLXh6sT82jrLKadycncFmPWgvLKtXkaeJXbq+kooq/LN7OvMSD9IppTe+2rSmvtFFeVU15lY2yShv5pZVUVNmIjwjg2XF96Rqt5RRU86WJX7m1rYfy+f3HG9mfW8x9ozrz0JhuWjJBtXia+JVbstkM767ezwvf7SQ80JePfnvBiSJqSrV0mviV28kqKOO/P93Eqj05XNk7mufG93O4no5SLYEmfuU2yiqrWbYziz8u3EpJRRXPjuvLpKHtdVSOcjua+FWLlV9ayYa0PNalHmX9/qNsTs+notpGz5jWvDppgNa7V25LE79qUZLSjrI4+TDrUvPYeaQAY8DLQ+gbG8ydI+IYEhfGxd0itGqmcmua+FWLsSU9n0lvr8XLQxjUIZSHRndjSHwoA9uH4u+jiV6p4zTxqxYhr7iCez5MIjLIly8euIgwvVir1Blp4lfNXrXN8OC8ZLILy/n0ngs16St1Dpr4VbP38tI9/Lg7m2fH9aV/+xBXh6NUk6e3KKpmbdnOTF5ZuocJg2OZNLS9q8NRqllwauIXkYdFZJuIbBWRj0XET0TiRWStiOwVkXkiot/LVb0cyC3hobnJ9IppzdM39tHx+Eo5yGmJX0TaAb8HEowxfQBPYCLwPPCiMaYLkAfc7awYVMtVVlnNPR8mAfDm7YO1Hr5SdeDsrh4vwF9EvIAAIAO4DJhv//37wI1OjkG1MMYY/rhwK9szCnhp4gA6hAe4OiSlmhWnJX5jzCFgJnAAK+HnA0nAMWNMlX2zdKBdba8XkWkikigiidnZ2c4KUzVDH687yPykdH4/uqvWxFeqHpzZ1RMKjAXigbZAIHCVo683xrxljEkwxiRERkY6KUrV3Gw8kMeMxdsY2S2SB0d3dXU4SjVLzuzqGQPsN8ZkG2Mqgc+AEUCIvesHIBY45MQYVAvyw/ZMbn9nLVGtfXn51gF4eujFXKXqw5mJ/wBwgYgEiDXcYjSwHVgOTLBvMxlY5MQYVAtgjOG15XuZ9kEinaOCmH/PcC2jrNR5cNoNXMaYtSIyH9gAVAEbgbeAr4C5IvK0fd27zopBuUZ5VTXeHh54NECLvLSimj8s2MwXmw4zdkBbnr+pn47gUeo8OfXOXWPMU8BTp6xOAYY6832V63yx6TCPfrqJYH9vru7Thqv7xjAkLqxe3TIZ+aVMm53E1sP5PH5VD+65pJOO1VeqAWjJBtUgjDG8uTKF57/dyeCOoUQG+TJ3/UHe/zmNiCAfruzdhmv6xjAsPgwvB+a0TUrLY/oHSZRVVvPOHQmM7qmjd5RqKJr41Xmrqrbxp0Xb+HjdAW7o35a/39wPXy9PisurWLErm6+3ZvDZhkPMWXuAsEAfLusRRWyoP6EBPoQEeBMa4HPyeaAP3249wpOfbaFNsB8fTR1Gt2idMEWphqSJX52XovIq7puzgZW7s7l/VBceubzbib79QF8vru0Xw7X9YiitqGbl7my+2ZrB0h2Z5JVUnnW/wzuH89pvBulFXKWcQBO/qreM/FLuei+R3ZmFPDe+LxOHdjjjtv4+nlzVpw1X9WkDQGW1jfzSSo6VVJBXUsnR4ooTzwN9PJk4tAPeDnQJKaXqThO/qpfthwu46731FJVXMWvKEEZ2q9tNdt6eHkQE+RIR5OukCJVSZ6KJX9XZj7uz+d2cDbTy8+LTey6kZ0xrV4eklKoDTfyqTgrKKrnvow3Ehvrz3p1DaRPs5+qQlFJ1pJ2oqk4+/CWNwrIqZt7cX5O+Us2UJn7lsLLKav6zej8Xd42gT7tgV4ejlKonTfzKYZ8mHiSnqIL7RnVxdShKqfOgiV85pLLaxpsrUxjUIYRh8WGuDkc1F8bAyhdgZnfY3sj1GCvLYOsCWHS/9d42W+O+fxOmF3eVQ77YdJhDx0r5yw29tV6OckxVBXzxIGz6CAKj4JM74IL74PK/gKe3c97TGEhPtN5z6wIoywdPX9j4AUT2hEseg143goeLC/2VFUDKckj7GVrHQMwAiOkH/qGN8vaa+NU52WyGN1bso0ebVlzWI8rV4bgfmw02z4Wk9yGmP/QeB+2HgUcDfWG32SBtNZQXQXQvCO5w/vsuPQbzbofUVXDpk3DRQ/D9n+CX1+BQItz8HrRu2yDhA5B/yDpGyR9D7h7w8oee18OA30DcRVaLf+ULMP8uiHgOLn4U+twEno2YAnP3we7vYPe3kLYGbJXg5QdVZSe3CekIbQdYf+eY/tYJITCiwUMRY0yD77ShJSQkmMTERFeH4ba+23aE6R8k8fLEAYwdUOtMmcpZ0n6Gb5+AjGQI7wL56VaiaBUDvcZardf6ngQKDsPGObBxNhw7cHK9TyuI6mmdBKJ62x97QYCDXXx5aTDnZjiaAmP/Bf0nnvzd1gWw+PdWwrvpHeg8qu5x15SxGZY9DXu+Bwx0GA4DJlnHxe+U+0tsNtixCFb+HbK2QVgn6wTQ75ZffwOproTKEqgstR79Qhz/7Kc6uA62LYQ930HuXmtdZA/odiV0uwpih1rfSjKSIWOT/ScZ8lJP7mP6j9ZJoB5EJMkYk3Daek386myMMdz4+hryiitY9t+XOFRZs1krKwBvf+d1RTgqLw2WPAXbPodWba3ukT4ToLLYajVu+xz2/ADV5SdPAt2vgfDO1vKZujKqq6wkuWG2lYyMDeJHwqDJVmszaxtkboPM7dbz0ryTr207EPreAn3GQ6s2te8/PQk+vhWqK+DWORB/8enbZO+2un2yd8KoJ63kW9cT17GDVsLfPA/8Q2DIVCvhh3U692ttNtj1lfUN4Mhm8A8DT5+Tid52Sh0pT18Y8SBc9DD4BDgWX0EGfP+/1onO09c6Dl2vhG5XQGjcuV9fmgdHtsDhZBg6DbzrN3RaE7+qlzV7c/jNO2t5ZlwfbhvW0dXhOEdRtpVIt3wK6eusdb7BVisvMAICwu0/YVbrz9PHOjF4eNkfvU8ue3gCZ7kG4tcagttD63a1/2cuL4TVL8Kaf1n7GvEgDH8AfAJr33bXt7B94cmTAIB4WvsPjrV+Qtpbj/mHIHkOFGZAUDQMuA0G/deZk6UxUHjEOhFkJMOOL6xH8bBOFn1vgZ7XgZ99aO+OL2HBbyEoCm77FCK7n/k4VBTDFw/Blk+gyxgY/7ZjreqyfFj1T/jlDWv5gnvgokes5F9Xxlgn0R2Lrb+dd4B10j/xaH++b6n1b6N1LFz5tPVt4kzXuaqrYP3bsOwZ6+R38SNw4f3gG1T3+BqAJn5VL7e98wu7M4tY9YdRLWvmq/JC2Pm1lXj2LQdTDdF9rH5h8YDiHCjJ/fVPcc7J5NoQAqNqJOcO4NsKEv8DRZnQbyKM/jMEO9i1VlZgdSvkH7S6g2o+FhwGW5X1ubpcDoMnQ9cr6vetJnu3lQS3fAp5+63WbPerrFbsT69Au8EwaS4EOVC7yRhImgXfPG4l3jZ97Rc57f3bkd1PxlhVAYnvWq300qPW8bnsj9ZJrTGkrYGv/wCZW6yT3tUvWN1hNR1cB18+Ym3TZQxc83fHvoE4kSZ+VWebDh5j7Gs/8eQ1PZg2srOrw6k/Y6yvzvkHrQtsO7+0kn5VqXUhs+8E6Huz1Zd9rv1UlVtdAdWVVjKtrrQvV1mPtuqz7cC66Jmfbv85UON5utXNEDsUrnoOYgc33Oe3VVstd09vqzXeEIyBQ0mw+RPY9hkUZ0PPG2D8W1ZLuS4yNlvfRDI2Wc8ri631nr7Qpo91QkhZaZ1o4i+BK/5W7z7v82Krtk7My562Gg5Dp8GlT1jdZUuesrrPWrez/n49rz/zt4JGpIlf1dn0DxL5JeUoPz1xGUG+zWAAmDHW1/JDG09v+VaWnNzOP8waGdPvFuvCaBP4D4oxUF4Avq2bRjx1UV1lXYwM63T+o4Fs1dbJ+fhFzuMng5AOMOYpqyXt6uNTchSW/Q0SZ1ldgMZm/e0u+B1c8rjLunVqc6bE3wz+NytX2JNZyHfbMvn96K5NP+lXV1mtztUvQtZ2a11gpNWFEtnd6t443qUSHGu1IF198fZUIif7ypsbTy+IaKC7uT08IbKb9dPvZmudMa5P9jUFhMF1L8LgKfDDU1YX2hVPn/sbYxPSxP9HK1d5Y+U+/L09uXN4nKtDObPKMkj+0OpbPpZmDZO78U1rhIujoy9U09eUkn5NMf3hjoWujqJeNPG3cGWV1Rw8WkJKTjGpOcXszykmJaeYYyUVxAT7ExvqT7tQf2JDA4gN9Sc2xJ/yKhuLkg8z+cK4pjn1YVk+rH/XGtlRnAXtEuCq/4NuVzfcTU1KtWDnTPwicj3wlTGmToUuRKQ7MK/Gqk7An4HZ9vVxQCpwizEm79TXq/rLLiznfz/fwvaMAg4dK6XmZZyIIB/iIwLpGB7IkfwyNqcfO23+WxHw8hCmjoxv5MiBo/utoYkVRSfHVVeW1nheAgfWQnk+dL7MGsoXd1HTbRUq1QQ50uK/FXhJRBYA/zHG7HRkx8aYXcAAABHxBA4BnwNPAEuNMc+JyBP25cfrE7yq3Rsr9rFsZxbX9othwuBY4iMCiY8IJC4ikNZ+p/dtF5dXcehYKel5JRzKK+VgXindo1sRE1zH0RnHFRy2Lnp51WFaxZy9sOof1g055vjIGKl9bHXXy2H4/dYNRUqpOjtn4jfG3C4irYFJwHsiYoBZwMfGmEIH32c0sM8YkyYiY4FL7evfB1agib/B5JdWMm/9Aa7rF8NLEx1LjIG+XnSLbkW36Fb1f+PjI2rW/MsqPuUbDD2usW526TzqzCeB7F3w40zYOt8avjdsuvUT1MZ6jbbklWpwDvXxG2MKRGQ+4A88BIwDHhORV4wxrzqwi4nAx/bn0caYDPvzI0B0bS8QkWnANIAOHTo4EqYC5q47QHFFNb+9uJFuHKkqhy3z4efXrFv8g9rApf9j3VK/8wvY9HHtJ4HMbfDj3606Jt7+1t2Nwx9ouHHmSqkzOuc4fhG5AbgT6ILVP/++MSZLRAKA7caYuHO83gc4DPQ2xmSKyDFjTEiN3+cZY85ai1TH8TumstrGyBeWExceyMfTLnDum5XmWeOY1/4bio5YRbyGP2DVk/GyXxCuqoD9K61yCDu/tC7K+gZDdG84sAZ8gqybYC68zykVCJVyd+czjv8m4EVjzI81VxpjSkTkbgdefzWwwRiTaV/OFJEYY0yGiMQAWQ7sQzngq80ZZOSX8cy4Ps59ox//DqtetO6w7DQKbnzdutB6areMl4/VH9/1cqh6CVJWWHVlDq6DkX+AC+6tf9VDpVS9OZL4ZwDHu2YQEX+s7ppUY8xSB14/iZPdPACLgcnAc/bHRp6Wp2UyxvD2qoWeY0sAABfwSURBVBQ6RwZyaTcndpdsX2Tdst7jOut29TZ9HXudl49VmbDbFc6LTSnlEEcGPX8K1BzKWW1fd04iEghcDnxWY/VzwOUisgcYY19W5+nnlFy2HS7gtxd3wsPDSRdECw5btdTbDbYm0nA06SulmhRHWvxexpiK4wvGmAp7v/05GWOKgfBT1uVijfJRDeidVfsJD/Rh3EAnTZRis8Hn91ilZse/3fRKHiilHOZIiz/bfoEXAPtwzBznhaTqam9WIct2ZnHHhXHOK538y+vWhdqr/s+a7EMp1Ww50uK/B5gjIv/CmmHiIHCHU6NSdfLu6v34enlw+wVOGvZ6ZAss/Qt0v9aaqUkp1aw5cgPXPuACEQmyLxc5PSrlsJyichZsOMSEwbGEB9XhTllHVZbCgqngHwo3vKo3VCnVAjh0A5eIXAv0BvzE/h/fGPNXJ8alHPTBz2lUVNm4+yIn1dVZMgOyd8DtCyAw/JybK6WaPkeKtL0JBACjgHeACcA6J8fl9g4eLcHfx5OIs7Tiyyqr+eCXNMb0jKJzpH3yB2Osu2hXvwgRXa1p4uIuhtghdZ+wee8SWPsmDLvHmgBDKdUiONLiH26M6Scim40xfxGRfwDfODswd7Y7s5AbX/sJmzHcPqwj0y/pTGSr008ACzakc7S44mR5hrJ8WPg76y7ZuIutCa1//DusfB68/KD9UIgbaZ0M2g06+8ic4hxrX5E9YcwMp3xOpZRrOJL4y+yPJSLSFsgFYpwXknvLL61k+gdJBPh4MbJrBP/5aT8frk3jjgvjmDay04lvADab4d1V++nbLphh8WHWBdhP7oC8NLjiGasMgoh1MkhbA/tXwf4fYfnTsBzrRBDZHaJ6WzMHRfe2nh+vlbP491ZZhtsX1H0OVaVUk+ZI4v9CREKAvwMbAAO87dSo3JTNZnhkXjIHj5bw8bQLGBIXxv2XdeHVZXt5Z1UKH/ycxh3DOzJ9ZGc2pOWRklPMyxMHIMlz4Kv/ti7ATvkKOl54cqd+wdD9ausHrPlCU1fDwbXWNIX7lsKmj05uHxBuzW96eKM1nZzepKVUi3PWIm0i4gFcYIxZY1/2BfyMMfmNFB/gPkXaXlqym5eW7OEvN/Rm8ilTHu7LLuLVpXtYtOkw/t6ehAb44G0rY1mvr/BI/hDiL4Gb3oWgyLq/cXGuVVkz0/6Ttd2aOHvcWzqjlVLN2JmKtDlSnXOjMcalM164Q+Jfsj2T385OZPygdvzj5v7IGYZN7s0q4pWle9i8eQOfRfybsMJdMPIxqxSyh5Nu3lJKNUvnU51zqYjcBHxmznWWUPWSkl3Ew/OS6dOuNc+O62sl/cpSOJoC+elw7ID1mJ9Ol/x0XslPx/gdhqpguG2+Vf1SKaUc5Ejinw48AlSJSBnW3bvGGNPaqZG5iaLyKqZ/kISXp/Dm7YOtkgvZu2D2jVB4+OSGHt4Q3A6C20P8xUhIBxh4u9Ufr5RSdeDInbvnMR+fOhtjDI99uol92UV8ePcwYkMDrNE5s28E8bCKoYXGQ0h7CIzS/nalVINw5AaukbWtP3ViFlV3b6zcxzdbj/DkNT0Y3iUCDiXBB+PBJxDuWAwRXVwdolKqBXKkq+exGs/9gKFAEnCZUyJyEyt3ZzPzu11c1y+GqRd3ggO/wIcTrBmpJi+G0DhXh6iUaqEc6eq5vuayiLQHXnJaRG5gzd4cpn+QSLfoVrwwoR+y/0f4eCK0bmu19IOdVFNfKaVwrB7/qdKBng0diLv4cXc2d763ng5hAXxw9zAC0pbDR7dYLfw7v9Gkr5RyOkf6+F/FulsXrBPFAKw7eFUdLd+ZxfQPk+gUEcic3w4j/OAP8OkUiOoJ/7VQq18qpRqFI338Ne+cqgI+Nsb85KR4Wqwftmdy35wNdI0O4sO7hxGashg+m2YVS7ttPviHuDpEpZSbcCTxzwfKjDHVACLiKSIBxpgS54bWcny7NYP7P9pI77atmX3XMIJ3zIEvHoKOI+A3c8FXR8wqpRqPI338S4Ga5Rn9gSXOCafl+XLzYe77aCP9YoP54LfDCE7+N3zxoFXf/rZPNekrpRqdIy1+v5rTLRpjikQkwIkxtRiLkg/x8LxkBncMZdaUIQT9bK+N3+tG6+YsLx9Xh6iUckOOtPiLRWTQ8QURGQyUOrJzEQkRkfkislNEdojIhSISJiI/iMge+2NofYNvyj5Zf5CH5yUzND6M9+8cQtCKP1tJf+DtMOE/mvSVUi7jSOJ/CPhURFaJyGpgHnC/g/t/GfjWGNMD6A/sAJ4AlhpjumJ1Iz1R97CbLmMMb67cxx8WbGZElwhm3TGYgG8fhl9eh2H3wvWvahVNpZRLOXID13oR6QF0t6/aZYypPNfrRCQYGAlMse+nAqgQkbHApfbN3gdWAI/XNfCmyGYzPPv1Dt5ZvZ/r+7flH+N74rN4Gmz7HC553CqdfIZyy0op1VjO2eIXkfuAQGPMVmPMViBIRH7nwL7jgWxglohsFJF3RCQQiDbGZNi3OQJEn+F9p4lIoogkZmdnO/ZpXKiy2sajn27indX7mXxhR14e1wWf+f9lJf0rnoZRT2rSV0o1CY509Uw1xhw7vmCMyQOmOvA6L2AQ8IZ9IpdiTunWsdf3r7XGvzHmLWNMgjEmITKyHrNKNaLSimqmzU7ks42H+L8Lbczw/A8eL/aCPT/AdS/B8AdcHaJSSp3gyKgeTxGR45OwiIgn4MiVyXQg3Riz1r48HyvxZ4pIjDEmQ0RigKz6BN5UHCup4IFZK+l4+Gs2RP1M2MYd1kTmvcbCkKnQfoirQ1RKqV9xJPF/C8wTkX/bl6cD35zrRcaYIyJyUES6G2N2AaOB7fafycBz9sdF9Yq8CcjZuZqkBf/k3xWrCfAuB/8+MOLv0O9ma+JzpZRqghxJ/I8D04B77MubgTYO7v8BYI6I+AApwJ1Y3UufiMjdQBpwS50ibiKObVxIxKLJXGT8KOh6IwGjpkPbQdqPr5Rq8hwZ1WMTkbVAZ6wkHQEscGTnxphk4LSJfrFa/82XMRT/8Bz5Jpqiycvp3Ukraiqlmo8zJn4R6QZMsv/kYI3fxxgzqnFCa7qKdq+kXckO5sc8zARN+kqpZuZsLf6dwCrgOmPMXgARebhRomricr//O2WmNb2vudfVoSilVJ2dbTjneCADWC4ib4vIaMDtO7ArM7bRMXc1K4LH0rNDrbcgKKVUk3bGxG+MWWiMmQj0AJZjlW6IEpE3ROSKxgqwqTn89QuUGh9ixjhatUIppZqWc97AZYwpNsZ8ZJ97NxbYSAspsVBXJv8QbQ9+yXe+lzO8b/dzv0AppZqgOs25a4zJs99R27xH5dTToW9fxMNU433RA4gO21RKNVP1mWzdPZUVELZzDss8LmTM8KGujkYppepNE7+Dslb8mwBTQt7Ae/H10rLKSqnmSxO/I6oq8En8N7+Y3lwx5ipXR6OUUudFE78DCtZ/TEhVNns630lIgM6cpZRq3jTxn4sxlP/4Ejtt7Rl5zSRXR6OUUudNE/85lG3/lsjSFH5ucxsdI4JcHY5SSp03R6pzurW8JTPBhNH/qrtcHYpSSjUIbfGfRfXBJGLyEvmu1XgGddLyDEqplkFb/GeR9e0LBBp/2o2+59wbK6VUM6Et/jMwRVlEHfqer3yu5LL+XVwdjlJKNRhN/GeQufMXPLHRut/1eHpoeQalVMuhif8McnZbc8T3GDDcxZEopVTD0sR/JhmbSCWG+HYxro5EKaUalCb+M4gs2kFmYA88tJtHKdXCaOKvRV72YaJNDtXR/VwdilJKNTinDucUkVSgEKgGqowxCSIShjVxexyQCtxijMlzZhx1tX/LGkKB0C5aflkp1fI0Rot/lDFmgDEmwb78BLDUGNMVWGpfblIKUhIBiO+rF3aVUi2PK7p6xgLv25+/D9zoghjOyjd7M0c82uDXKszVoSilVINzduI3wPcikiQi0+zroo0xGfbnR4BaayGIyDQRSRSRxOzsbCeHeVJZZTWxZXvIC+ndaO+plFKNydmJ/yJjzCDgauA+ERlZ85fGGIN1cjiNfW7fBGNMQmRkpJPDPGlHShrtJQvPdgMa7T2VUqoxOTXxG2MO2R+zgM+BoUCmiMQA2B+znBlDXaVv/wWANt2HuTgSpZRyDqclfhEJFJFWx58DVwBbgcXAZPtmk4FFzoqhPsoPbACgdachLo5EKaWcw5nDOaOBz0Xk+Pt8ZIz5VkTWA5+IyN1AGnCLE2OoE2MMrfK2cdS7DWEBemFXKdUyOS3xG2NSgP61rM8FRjvrfc/Hvuxiutn2URLWB037SqmWSu/crWHz3jTiPTLxjxvs6lCUUsppNPHXkLl7PQBhnbV/XynVcmnir8Ec3gSAtNWhnEqplksTv11OUTltS3dS5BsFQVGuDkcppZxGE79dUloefSSVqiityKmUatk08dtt3neQTpJBULxe2FVKtWxOLcvcnOSlbMRDDB7tBrk6FKWUcipt8WMVZgvI3WItxJx264FSSrUomviBzen59GQ/5X6R0Frn2FVKtWya+IHEtKP0llSkrbb2lVItnyZ+YHNKBl09DuETq/37SqmWz+0Tv81mKDqwCU9s2r+vlHILbp/492UXEV+5x1rQO3aVUm7A7RN/ov3GrWq/MGjdztXhKKWU07l94l+fepQBXql4tBsI1twBSinVorl94t+SmkkXDiLav6+UchNunfizC8vxz9uFJ9Xav6+UchtunfiT0o7SxyPVWtAWv1LKTbh14k9MzaO/536MXwiEdHR1OEop1SjcOvGvT8sjwcfev68XdpVSbsJtE39BWSW7DuXSsTpV+/eVUm7FbRP/L/ty6WwO4mUqIUYTv1LKfTg98YuIp4hsFJEv7cvxIrJWRPaKyDwR8XF2DLVZvTeHQd6p1oJe2FVKuZHGaPE/COyosfw88KIxpguQB9zdCDGcZtWeHEYFZ4BvMIR1ckUISinlEk5N/CISC1wLvGNfFuAyYL59k/eBG50ZQ23S80rYn1NMX9kPMf30wq5Syq04u8X/EvAHwGZfDgeOGWOq7MvpQK0FckRkmogkikhidnZ2gwa1ek8OIRQSUbQTYoc06L6VUqqpc1riF5HrgCxjTFJ9Xm+MecsYk2CMSYiMjGzQ2FbtyeGWwGTEVgW9xjbovpVSqqlz5mTrI4AbROQawA9oDbwMhIiIl73VHwsccmIMp6m2GX7al8Mn/uvBt5Ne2FVKuR2ntfiNMf9jjIk1xsQBE4FlxpjbgOXABPtmk4FFzoqhNtsO5+NZkkPX4g3Qe7z27yul3I4rxvE/DjwiInux+vzfbcw3X7Unh6s91yHYoM/4xnxrpZRqEpzZ1XOCMWYFsML+PAUY2hjvW5tVe7J50n89hHSHqF6uCkMppVzGre7cLamo4mDaPvpWbYPe47SbRynlltwq8a9NOcoY1iEY7eZRSrktt0r8q/bkcIPXL9iiekFkd1eHo5RSLuFWiX/X7u0Mll14aGtfKeXG3CbxH8kvo8fRZdZCb038Sin35TaJf/XeHK73/IXSiL4Q3tnV4SillMu4TeLfvm0zAzz24dt/wrk3VkqpFswtEr/NZghJ/QoAjz6NXgxUKaWaFLdI/DuPFDKqajVHQ/pCaJyrw1FKKZdyi8S/aVMSfT1S8dZuHqWUco/E77VzIQCtBmniV0qpFp/4yyqr6XdsGQcC+0FwrKvDUUopl2vxiX/rpnV0lwOU9dAJV5RSCtwg8ZdunI/NCLEjJro6FKWUahJaduI3ho4Z37HTtw8BYdrNo5RS0MIT/9H9yXSwHSSr47WuDkUppZqMFp34s3/5mGojRA692dWhKKVUk9GiE3/ZkV2slz706Ky1eZRS6rhGmXrRVX4a9A9KS4q4wENn2lJKqeNadOL/3aVdXB2CUko1OS26q0cppdTpNPErpZSbcVriFxE/EVknIptEZJuI/MW+Pl5E1orIXhGZJyI+zopBKaXU6ZzZ4i8HLjPG9AcGAFeJyAXA88CLxpguQB5wtxNjUEopdQqnJX5jKbIvett/DHAZMN++/n1AZ0ZRSqlG5NQ+fhHxFJFkIAv4AdgHHDPGVNk3SQfaneG100QkUUQSs7OznRmmUkq5FacmfmNMtTFmABALDAV61OG1bxljEowxCZGRkU6LUSml3E2jjOoxxhwDlgMXAiEicvz+gVjgUGPEoJRSyiLGGOfsWCQSqDTGHBMRf+B7rAu7k4EFxpi5IvImsNkY8/o59pUNpJ3h1xFATgOG3pA0tvrR2OpHY6uflhxbR2PMaV0mzkz8/bAu3npifbP4xBjzVxHpBMwFwoCNwO3GmPLzeJ9EY0xCQ8Tc0DS2+tHY6kdjqx93jM1pJRuMMZuBgbWsT8Hq71dKKeUCeueuUkq5mZaQ+N9ydQBnobHVj8ZWPxpb/bhdbE7r41dKKdU0tYQWv1JKqTrQxK+UUm6mWSd+EblKRHbZK30+4ep4ahKRVBHZIiLJIpLo4lj+IyJZIrK1xrowEflBRPbYH0ObUGwzROSQ/dgli8g1LoqtvYgsF5Ht9gqzD9rXu/zYnSU2lx+7plyZ9yyxvSci+2sctwGNHVuNGD1FZKOIfGlfbvjjZoxplj9Y9wfsAzoBPsAmoJer46oRXyoQ4eo47LGMBAYBW2usewF4wv78CeD5JhTbDODRJnDcYoBB9uetgN1Ar6Zw7M4Sm8uPHSBAkP25N7AWuAD4BJhoX/8mcG8Tiu09YIKr/83Z43oE+Aj40r7c4MetObf4hwJ7jTEpxpgKrJvCxro4pibJGPMjcPSU1WOxbrADF1ZJPUNsTYIxJsMYs8H+vBDYgVVU0OXH7iyxuZyxNMnKvGeJrUkQkVjgWuAd+7LghOPWnBN/O+BgjeUzVvp0EQN8LyJJIjLN1cHUItoYk2F/fgSIdmUwtbhfRDbbu4Jc0g1Vk4jEYd2QuJYmduxOiQ2awLE7n8q8jR2bMeb4cXvGftxeFBFfV8QGvAT8AbDZl8NxwnFrzom/qbvIGDMIuBq4T0RGujqgMzHWd8gm0+oB3gA6Y03gkwH8w5XBiEgQsAB4yBhTUPN3rj52tcTWJI6dOY/KvM52amwi0gf4H6wYh2CVk3m8seMSkeuALGNMkrPfqzkn/kNA+xrLTarSpzHmkP0xC/icplemIlNEYgDsj1kujucEY0ym/T+nDXgbFx47EfHGSqxzjDGf2Vc3iWNXW2xN6djZ42mylXlrxHaVvevMGKtu2Cxcc9xGADeISCpW1/VlwMs44bg158S/Huhqv+LtA0wEFrs4JgBEJFBEWh1/DlwBbD37qxrdYqxKqdgfF7kwll85nlTtxuGiY2fvX30X2GGM+WeNX7n82J0ptqZw7EQkUkRC7M/9gcuxrkEsBybYN3PVcasttp01TuSC1Yfe6MfNGPM/xphYY0wcVj5bZoy5DWccN1dfwT7Pq9/XYI1m2Af8r6vjqRFXJ6xRRpuAba6ODfgY62t/JVYf4d1YfYdLgT3AEiCsCcX2AbAF2IyVZGNcFNtFWN04m4Fk+881TeHYnSU2lx87oB9W5d3NWAn0z/b1nYB1wF7gU8C3CcW2zH7ctgIfYh/546of4FJOjupp8OOmJRuUUsrNNOeuHqWUUvWgiV8ppdyMJn6llHIzmviVUsrNaOJXSik3o4lfKUBEqmtUZkyWBqz2KiJxNauPKuVqTptsXalmptRYt/Er1eJpi1+psxBrXoUXxJpbYZ2IdLGvjxORZfaiXktFpIN9fbSIfG6v975JRIbbd+UpIm/ba8B/b79rVCmX0MSvlMX/lK6eW2v8Lt8Y0xf4F1b1RIBXgfeNMf2AOcAr9vWvACuNMf2x5hnYZl/fFXjNGNMbOAbc5OTPo9QZ6Z27SgEiUmSMCaplfSpwmTEmxV4U7YgxJlxEcrDKIVTa12cYYyJEJBuINVaxr+P7iMMq/9vVvvw44G2Medr5n0yp02mLX6lzM2d4XhflNZ5Xo9fXlAtp4lfq3G6t8fiz/fkarAqKALcBq+zPlwL3wokJP4IbK0ilHKWtDqUs/vZZmY771hhzfEhnqIhsxmq1T7KvewCYJSKPAdnAnfb1DwJvicjdWC37e7GqjyrVZGgfv1JnYe/jTzDG5Lg6FqUainb1KKWUm9EWv1JKuRlt8SullJvRxK+UUm5GE79SSrkZTfxKKeVmNPErpZSb+X/rC+xBzqbXZQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uc4YyHsnd4ef",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "932f9376-bb07-4710-d126-53a9549e7799"
      },
      "source": [
        "with torch.set_grad_enabled(False):\n",
        "    test_acc = compute_acc(model=model,\n",
        "                           data_loader=test_loader,\n",
        "                           device=DEVICE)\n",
        "    \n",
        "    valid_acc = compute_acc(model=model,\n",
        "                            data_loader=valid_loader,\n",
        "                            device=DEVICE)\n",
        "    \n",
        "\n",
        "print(f'Validation ACC: {valid_acc:.2f}%')\n",
        "print(f'Test ACC: {test_acc:.2f}%')"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Validation ACC: 73.30%\n",
            "Test ACC: 72.33%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g0ip2kUQd4eh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "outputId": "fdd3394f-ebb2-437d-f23b-f410428058d8"
      },
      "source": [
        "%watermark -iv"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch     1.5.1+cu101\n",
            "PIL.Image 7.0.0\n",
            "numpy     1.18.5\n",
            "pandas    1.0.5\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}